Tuesday, July 4, 2017

How to fetch WWDC 2017 Video Subtitle to SRT format

Create and Run this script wwdc2017_fetch_srt.sh to fetch WWDC2017 subtitle
Reference : https://github.com/wsvn53/wwdc2016-subtitles

wwdc2017_fetch_srt.sh    Select all
#!/bin/sh # @Author: Ethan # @Date: 2016-06-22 14:10:53 # @Last Modified by: javacom # @Last Modified time: 2017-07-04 WWDC_YEAR=2017; WWDC_SESSION_PREFIX=https://developer.apple.com/videos/play/wwdc$WWDC_YEAR; WWDC_LOCAL_DIR=$(basename $WWDC_SESSION_PREFIX); detect_video_m3u8 () { local session_url=$WWDC_SESSION_PREFIX/$SESSION_ID/; local session_html=$(curl -s $session_url); local video_url=$(echo "$session_html" | grep .m3u8 | grep $SESSION_ID | head -n1 | sed "s#.*\"\(https://.*m3u8\)\".*#\1#"); echo "$session_html" | grep .mp4 | grep $SESSION_ID | sed "s#.*\"\(https://.*mp4\).*\".*#\1#" | while read mp4_url; do local mp4_filename=$(basename $mp4_url | cut -d. -f1); local srt_filename=$mp4_filename.srt; echo "> Subtitle local: $WWDC_LOCAL_DIR/$srt_filename" >&2; > $WWDC_LOCAL_DIR/$srt_filename; done echo "$video_url"; echo "> Video: $video_url" >&2; } detect_subtitle_m3u8 () { local video_url=$1; local subtitle_uri=$(curl -s $video_url | grep "LANGUAGE=\"eng\"" | sed "s#.*URI=\"\(.*\)\"#\1#"); local subtitle_url=$subtitle_uri; [[ "$subtitle_uri" != http* ]] && { subtitle_url=$(dirname $video_url)/$subtitle_uri; } echo "$subtitle_url"; echo "> Subtitle: $subtitle_url" >&2; } download_subtitle_contents () { local subtitle_url=$1; echo "> Downloading... " local subtitle_base_url=$(dirname $subtitle_url); curl -s $subtitle_url | grep "webvtt" | while read webvtt; do local subtitle_webvtt=$subtitle_base_url/$webvtt; #echo "- get $subtitle_webvtt"; local subtitle_content=$(curl -s $subtitle_webvtt); ls $WWDC_LOCAL_DIR/"$SESSION_ID"_* | while read srt_file; do echo "$subtitle_content" >> $srt_file; done done } main () { [ ! -d $WWDC_LOCAL_DIR ] && { mkdir $WWDC_LOCAL_DIR; } curl -s $WWDC_SESSION_PREFIX | grep /videos/play/wwdc$WWDC_YEAR | sed "s#.*/videos/play/wwdc$WWDC_YEAR/\([0-9]\{3\}\).*#\1#" | sort | uniq | while read SESSION_ID; do #echo "SESSION_ID is" $SESSION_ID local video_url=$(detect_video_m3u8 $SESSION_ID); local subtitle_url=$(detect_subtitle_m3u8 $video_url); download_subtitle_contents $subtitle_url; done } main;




Run this shell script to format as SRT subtitle

shellscript.sh    Select all
cd wwdc2017 mkdir -p sd mkdir -p hd for i in ???_sd_*.srt; do sed -e '/WEBVTT/d;/X-TIMESTAMP/d;' $i | awk '/^[0-9]{2}:[0-9]{2}:/ {seen[$0]++; skipduplicated=0} {if (seen[$0]>1) skipduplicated=1; if (!skipduplicated) print $0}' | awk -v RS="" '{gsub("\n", "-Z"); print}' | awk '$0 !~/^WEB/ {print $0}' | uniq | awk '{printf "\n%s-Z%s", NR,$0 }' | awk -v ORS="\n\n" '{gsub("-Z", "\n"); print}' | sed -e 's/.A:middle$//g;s/&gt;/>/g;s/&lt;/</g;1,2d;' > sd/$i; done for i in ???_hd_*.srt; do sed -e '/WEBVTT/d;/X-TIMESTAMP/d;' $i | awk '/^[0-9]{2}:[0-9]{2}:/ {seen[$0]++; skipduplicated=0} {if (seen[$0]>1) skipduplicated=1; if (!skipduplicated) print $0}' | awk -v RS="" '{gsub("\n", "-Z"); print}' | awk '$0 !~/^WEB/ {print $0}' | uniq | awk '{printf "\n%s-Z%s", NR,$0 }' | awk -v ORS="\n\n" '{gsub("-Z", "\n"); print}' | sed -e 's/.A:middle$//g;s/&gt;/>/g;s/&lt;/</g;1,2d;' > hd/$i; done




Run this script wwdc2017_fetch_mp4.sh to download all mp4 videos

wwdc2017_fetch_mp4.sh    Select all
#!/bin/sh # @Last Modified by: javacom # @Last Modified time: 2016-07-08 WWDC_YEAR=2017; WWDC_SESSION_PREFIX=https://developer.apple.com/videos/play/wwdc$WWDC_YEAR; WWDC_LOCAL_DIR=$(basename $WWDC_SESSION_PREFIX); download_mp4_video () { local session_url=$WWDC_SESSION_PREFIX/$SESSION_ID/; local session_html=$(curl -s $session_url); local video_url=$(echo "$session_html" | grep .m3u8 | grep $SESSION_ID | head -n1 | sed "s#.*\"\(https://.*m3u8\)\".*#\1#"); echo "$session_html" | grep .mp4 | grep $SESSION_ID | sed "s#.*\"\(https://.*mp4\).*\".*#\1#" | while read mp4_url; do local mp4_filename=$(basename $mp4_url); if [ -e $WWDC_LOCAL_DIR/$mp4_filename ] then echo "> MP4 already existed : $WWDC_LOCAL_DIR/$mp4_filename" >&2; else echo "> MP4 Downloading... : $mp4_url" >&2; curl -o $WWDC_LOCAL_DIR/$mp4_filename $mp4_url fi done } main () { [ ! -d $WWDC_LOCAL_DIR ] && { mkdir $WWDC_LOCAL_DIR; } curl -s $WWDC_SESSION_PREFIX | grep /videos/play/wwdc$WWDC_YEAR | sed "s#.*/videos/play/wwdc$WWDC_YEAR/\([0-9]\{3\}\).*#\1#" | sort | uniq | while read SESSION_ID; do download_mp4_video $SESSION_ID; done } main;


One line version wwdc2017_fetch_mp4.sh to download all mp4 videos

wwdc2017_fetch_mp4.sh    Select all
# one liner for hd videos download WWDCYEAR="wwdc2017"; for i in `curl -s https://developer.apple.com/videos/$WWDCYEAR/ | grep -o '<a href="/videos/play/'"$WWDCYEAR"'/[0-9]*' | cut -d '"' -f2 | sort | uniq`; do video_url=$(curl -s https://developer.apple.com${i} | grep -o 'http.*_hd_.*.mp4'); if [ ! -z "$video_url" ]; then mp4_filename=$(basename $video_url); if [ -e $mp4_filename ]; then echo "skipping $mp4_filename"; else echo "Downloading ... $mp4_filename";curl -O $video_url; fi; fi; done # one liner for sd videos download WWDCYEAR="wwdc2017"; for i in `curl -s https://developer.apple.com/videos/$WWDCYEAR/ | grep -o '<a href="/videos/play/'"$WWDCYEAR"'/[0-9]*' | cut -d '"' -f2 | sort | uniq`; do video_url=$(curl -s https://developer.apple.com${i} | grep -o 'http.*_sd_.*.mp4'); if [ ! -z "$video_url" ]; then mp4_filename=$(basename $video_url); if [ -e $mp4_filename ]; then echo "skipping $mp4_filename"; else echo "Downloading ... $mp4_filename";curl -O $video_url; fi; fi; done




Wednesday, June 7, 2017

How to train dataset in python and convert to CoreML model for iOS11

Reference http://machinelearningmastery.com/machine-learning-in-python-step-by-step/

Environment : macOS 10.12.4
matplotlib==2.0.0
numpy==1.12.1
pandas==0.19.2
scikit-learn==0.18.1
scipy==0.19.0
six==1.10.0
sklearn==0.18.1
coremltools==0.3.0
protobuf==3.3.0

Upgrade the pip and install the following python packages
shellscript.sh    Select all
pip install --upgrade pip sudo -H pip install numpy scipy matplotlib pandas sklearn coremltools protobuf



Convert to Core ML Run the following python code to show machine learning in python step by step and finally generate iris_lr.mlmodel
iris_learn.py    Select all
#!/usr/bin/env python # Check the versions of libraries # Python version import sys print('Python: {}'.format(sys.version)) # scipy import scipy print('scipy: {}'.format(scipy.__version__)) # numpy import numpy print('numpy: {}'.format(numpy.__version__)) # matplotlib import matplotlib print('matplotlib: {}'.format(matplotlib.__version__)) # pandas import pandas print('pandas: {}'.format(pandas.__version__)) # scikit-learn import sklearn print('sklearn: {}'.format(sklearn.__version__)) # Load libraries import pandas from pandas.tools.plotting import scatter_matrix import matplotlib.pyplot as plt from sklearn import model_selection from sklearn.metrics import classification_report from sklearn.metrics import confusion_matrix from sklearn.metrics import accuracy_score from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from sklearn.neighbors import KNeighborsClassifier from sklearn.discriminant_analysis import LinearDiscriminantAnalysis from sklearn.naive_bayes import GaussianNB from sklearn.svm import SVC # Load dataset url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data" names = ['sepal-length', 'sepal-width', 'petal-length', 'petal-width', 'class'] dataset = pandas.read_csv(url, names=names) # shape print(dataset.shape) # head print(dataset.head(20)) # descriptions print(dataset.describe()) # class distribution print(dataset.groupby('class').size()) # box and whisker plots dataset.plot(kind='box', subplots=True, layout=(2,2), sharex=False, sharey=False) plt.suptitle("Box and Whisker Plots for inputs") plt.show() # histograms dataset.hist() plt.suptitle('Histograms for inputs') plt.show() # scatter plot matrix scatter_matrix(dataset) plt.suptitle('Scatter Plot Matrix for inputs') plt.show() # Split-out validation dataset array = dataset.values X = array[:,0:4] Y = array[:,4] validation_size = 0.20 seed = 7 X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(X, Y, test_size=validation_size, random_state=seed) # Test options and evaluation metric seed = 7 scoring = 'accuracy' # Spot Check Algorithms models = [] models.append(('LR', LogisticRegression())) models.append(('LDA', LinearDiscriminantAnalysis())) models.append(('KNN', KNeighborsClassifier())) models.append(('CART', DecisionTreeClassifier())) models.append(('NB', GaussianNB())) models.append(('SVM', SVC())) # evaluate each model in turn results = [] names = [] for name, model in models: kfold = model_selection.KFold(n_splits=10, random_state=seed) cv_results = model_selection.cross_val_score(model, X_train, Y_train, cv=kfold, scoring=scoring) results.append(cv_results) names.append(name) msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std()) print(msg) # Compare Algorithms fig = plt.figure() fig.suptitle('Algorithm Comparison') ax = fig.add_subplot(111) plt.boxplot(results) ax.set_xticklabels(names) plt.show() # Make predictions on validation dataset knn = KNeighborsClassifier() knn.fit(X_train, Y_train) predictions = knn.predict(X_validation) print(accuracy_score(Y_validation, predictions)) print(confusion_matrix(Y_validation, predictions)) print(classification_report(Y_validation, predictions)) print("Make predictions on LogisticRegression Model") model = LogisticRegression() model.fit(X_train, Y_train) predictions = model.predict(X_validation) print(accuracy_score(Y_validation, predictions)) print(confusion_matrix(Y_validation, predictions)) print(classification_report(Y_validation, predictions)) # print prediction results on test data for i, prediction in enumerate(predictions): print 'Predicted: %s, Target: %s %s' % (prediction, Y_validation[i], '' if prediction==Y_validation[i] else '(WRONG!!!)') #convert and save scikit.learn model #support LogisticRegression of scikit.learn print("Convert LogisticRegression Model to coreml model") import coremltools coreml_model = coremltools.converters.sklearn.convert(model, ["sepal-length", "sepal-width", "petal-length", "petal-width"], "class") #set model metadata coreml_model.author = 'Author' coreml_model.license = 'BSD' coreml_model.short_description = 'LogisticRegression on Iris flower data set' #set features description manually coreml_model.input_description['sepal-length'] = 'Sepal Length in centimetres' coreml_model.input_description['sepal-width'] = 'Sepal Width in centimetres' coreml_model.input_description['petal-length'] = 'Petal Length in centimetres' coreml_model.input_description['petal-width'] = 'Petal Width in centimetres' #set the ouput description coreml_model.output_description['class'] = 'Distinguish the species' #save the model coreml_model.save('iris_lr.mlmodel') from coremltools.models import MLModel model = MLModel('iris_lr.mlmodel') #get the spec of the model print(model.get_spec())


Download Xcode 9 beta and the sample code from Apple

https://docs-assets.developer.apple.com/published/51ff0c1668/IntegratingaCoreMLModelintoYourApp.zip
Modify it and add the model to the xcode project


Try the new refactoring tool in Xcode 9. It is amazing.


Train data using Neural Network Model Keras
Reference : http://machinelearningmastery.com/5-step-life-cycle-neural-network-models-keras/

shellscript.sh    Select all
# download training data curl -O http://archive.ics.uci.edu/ml/machine-learning-databases/pima-indians-diabetes/pima-indians-diabetes.data # install and activate virtual environment and install necessary python packages # use deactivate to stop the python virtual env sudo -H pip install --upgrade virtualenv virtualenv --system-site-packages ~/tensorflow source ~/tensorflow/bin/activate # macOS, CPU only non-optimised, Python 2.7: # https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.1.0-py2-none-any.whl # macOS, GPU enabled, Python 2.7: # https://storage.googleapis.com/tensorflow/mac/gpu/tensorflow_gpu-1.1.0-py2-none-any.whl # or find optimised wheel files from the community https://github.com/yaroslavvb/tensorflow-community-wheels/issues # this optimised one (SSE4.1,SSE4.2,AVX,AVX2,FMA) works for Python 2.7 macOS 10.12 Tensoflow 1.1.0 CPU https://github.com/fdalvi/tensorflow-builds # this one works for GeForce GT 650M GPU and CPU (SSE4.2, AVX) and CUDA 8.0, and cuDNN v5.1 https://github.com/bodak/tensorflow-wheels/releases/tag/v1.1.0_27 # instruction to build your own python package https://ctmakro.github.io/site/on_learning/tf1c.html # suppose, install the official non-optimised wheel file as below pip install --upgrade https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.1.0-py2-none-any.whl pip install coremltools protobuf pip install keras==1.2.2 h5py

Convert to Core ML Run the following python code in virtual environment (tensorflow) to generate pima_keras.mlmodel
keras_learn.py    Select all
#!/usr/bin/env python from keras.models import Sequential from keras.layers import Dense import numpy # fix random seed for reproducibility numpy.random.seed(7) # load pima indians dataset #dataset = numpy.loadtxt("pima-indians-diabetes.csv", delimiter=",") dataset = numpy.loadtxt("pima-indians-diabetes.data", delimiter=",") # split into input (X) and output (Y) variables X = dataset[:,0:8] Y = dataset[:,8] # create model model = Sequential() model.add(Dense(12, input_dim=8, activation='relu')) model.add(Dense(1, activation='sigmoid')) # Compile model model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # Fit the model #model.fit(X, Y, epochs=150, batch_size=10) model.fit(X, Y, 10, 150) # parameters change to keras 1.2.2 # evaluate the model scores = model.evaluate(X, Y) print("\n%s: %.2f%%" % (model.metrics_names[1], scores[1]*100)) #convert and save keras model model.save('pima.h5') print("Convert Model to coreml model") import coremltools coreml_model = coremltools.converters.keras.convert('pima.h5') #set model metadata coreml_model.author = 'Author' coreml_model.license = 'BSD' coreml_model.short_description = 'pima-indians-diabetes' #save the model coreml_model.save('pima_keras.mlmodel') from coremltools.models import MLModel mlmodel = MLModel('pima_keras.mlmodel') #get the spec of the model print(mlmodel.get_spec())


Note: coremltools require python 2.7 (not for 3.x) and supports keras==1.2.2 with Tenorflow (1.0.x, 1.1.x) only. Tenorflow_gpu requires Nvidia Cuda 8.0 and cuDNN v5.1 (which also requires macOS 10.11/10.12) but recent models of Mac are all bundled AMD GPUs. Unless you could get an old Mac Pro with upgraded Nvidia GPU with at least 4 GB of video RAM, it is better to stay with Mac CPU i7 or get a Linux machine for data training purpose only.

Hardware reference for Linux : https://www.oreilly.com/learning/build-a-super-fast-deep-learning-machine-for-under-1000

For Windows PC, tensorflow/tensorflow_gpu is only available for Python 3.5 and 64 bits only as below. As current coremltools keras convertors are not compatible with python 3.5, so direct conversion is not available in PC yet.
https://storage.googleapis.com/tensorflow/windows/cpu/tensorflow-1.1.0-cp35-cp35m-win_amd64.whl
https://storage.googleapis.com/tensorflow/windows/gpu/tensorflow_gpu-1.1.0-cp35-cp35m-win_amd64.whl



keras-inception-test Run the following python code in virtual environment (tensorflow) to test Keras Inceptionv3 model. This will download the trained Inception V3 weights from https://github.com/fchollet/deep-learning-models/releases/download/v0.2/inception_v3_weights_tf_dim_ordering_tf_kernels.h5
shellscript.sh    Select all
git clone git://github.com/vml-ffleschner/coremltools-keras-inception-test cd coremltools-keras-inception-test/ # based on the created virtualenv in ~/tensorflow as above source ~/tensorflow/bin/activate # additional installation of packages pip install olefile pillow #Add coreml_model.author = 'Author' coreml_model.license = 'BSD' coreml_model.short_description = 'Image InceptionV3 model' coreml_model.save('Inceptionv3.mlmodel') print("CoreML model file Created") #After #print("CoreML Converted") #in playground.py # note : coreml_model.predict requires macOS 10.13 High Sierra python playground.py


Install tensorflow 1.1.0 library for Java is here
shellscript.sh    Select all
curl -O https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.1.0.jar curl -O https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-darwin-x86_64-1.1.0.tar.gz
# install tar xzvf libtensorflow_jni-cpu-darwin-x86_64-1.1.0.tar.gz -C ./jni # compile and run HelloTF javac -cp libtensorflow-1.1.0.jar HelloTF.java java -cp libtensorflow-1.1.0.jar:. -Djava.library.path=./jni HelloTF


Monday, November 28, 2016

How to build c++ libraries (boost & QuantLib) for iOS XCode 8

(1) Download boost 1.62.0 and extract source
shell script    Select all
# download boost source cd $HOME/Downloads curl -O https://nchc.dl.sourceforge.net/project/boost/boost/1.62.0/boost_1_62_0.tar.bz2 tar xjvf boost_1_62_0.tar.bz2 cd boost_1_62_0 # create the following user-config.jam at ~/Downloads/boost_1_62_0/tools/build/src/ cat > tools/build/src/user-config.jam << 'EOF' # ------------------ # darwin iphone configuration. # ------------------ local XCODE_ROOT = /Applications/Xcode.app/Contents/Developer ; using darwin : armv7 : $(XCODE_ROOT)/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang++ : <striper>$(XCODE_ROOT)/Toolchains/XcodeDefault.xctoolchain/usr/bin/strip <root>$(XCODE_ROOT)/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk <compileflags>"-isysroot $(XCODE_ROOT)/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk" <compileflags>"-arch armv7" <compileflags>-fexceptions <compileflags>-frtti <compileflags>-fpic <compileflags>-DBOOST_SP_USE_SPINLOCK <compileflags>-std=c++11 <compileflags>-stdlib=libc++ <compileflags>-miphoneos-version-min=7.0 <compileflags>-fvisibility=hidden <compileflags>-fvisibility-inlines-hidden <compileflags>-O2 <compileflags>-g <compileflags>-Os <compileflags>-Wno-long-long <compileflags>-Wno-missing-field-initializers <compileflags>-Wno-unused-variable <linkflags>-stdlib=libc++ <architecture>arm <address-model>32 <target-os>iphone ; using darwin : armv7s : $(XCODE_ROOT)/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang++ : <striper>$(XCODE_ROOT)/Toolchains/XcodeDefault.xctoolchain/usr/bin/strip <root>$(XCODE_ROOT)/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk <compileflags>"-isysroot $(XCODE_ROOT)/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk" <compileflags>"-arch armv7s" <compileflags>-fexceptions <compileflags>-frtti <compileflags>-fpic <compileflags>-DBOOST_SP_USE_SPINLOCK <compileflags>-std=c++11 <compileflags>-stdlib=libc++ <compileflags>-miphoneos-version-min=7.0 <compileflags>-fvisibility=hidden <compileflags>-fvisibility-inlines-hidden <compileflags>-O2 <compileflags>-g <compileflags>-Os <compileflags>-Wno-long-long <compileflags>-Wno-missing-field-initializers <compileflags>-Wno-unused-variable <linkflags>-stdlib=libc++ <architecture>arm <address-model>32 <target-os>iphone ; using darwin : arm64 : $(XCODE_ROOT)/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang++ : <striper>$(XCODE_ROOT)/Toolchains/XcodeDefault.xctoolchain/usr/bin/strip <root>$(XCODE_ROOT)/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk <compileflags>"-isysroot $(XCODE_ROOT)/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk" <compileflags>"-arch arm64" <compileflags>-fexceptions <compileflags>-frtti <compileflags>-fpic <compileflags>-DBOOST_SP_USE_SPINLOCK <compileflags>-std=c++11 <compileflags>-stdlib=libc++ <compileflags>-miphoneos-version-min=7.0 <compileflags>-fvisibility=hidden <compileflags>-fvisibility-inlines-hidden <compileflags>-O2 <compileflags>-g <compileflags>-Os <compileflags>-Wno-long-long <compileflags>-Wno-missing-field-initializers <compileflags>-Wno-unused-variable <linkflags>-stdlib=libc++ <architecture>arm <address-model>64 <target-os>iphone ; using darwin : i386 : $(XCODE_ROOT)/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang++ : <striper>$(XCODE_ROOT)/Toolchains/XcodeDefault.xctoolchain/usr/bin/strip <root>$(XCODE_ROOT)/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk <compileflags>"-arch i386" <compileflags>"-isysroot $(XCODE_ROOT)/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk" <compileflags>-fexceptions <compileflags>-frtti <compileflags>-fpic <compileflags>-DBOOST_SP_USE_SPINLOCK <compileflags>-std=c++11 <compileflags>-stdlib=libc++ <compileflags>-miphoneos-version-min=7.0 <compileflags>-fvisibility=hidden <compileflags>-fvisibility-inlines-hidden <compileflags>-O2 <compileflags>-g <compileflags>-Os <compileflags>-Wno-long-long <compileflags>-Wno-missing-field-initializers <compileflags>-Wno-unused-variable <linkflags>-stdlib=libc++ <architecture>x86 <address-model>32 <target-os>iphone ; using darwin : x86_64 : $(XCODE_ROOT)/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang++ : <striper>$(XCODE_ROOT)/Toolchains/XcodeDefault.xctoolchain/usr/bin/strip <root>$(XCODE_ROOT)/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk <compileflags>"-arch x86_64" <compileflags>"-isysroot $(XCODE_ROOT)/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk" <compileflags>-fexceptions <compileflags>-frtti <compileflags>-fpic <compileflags>-DBOOST_SP_USE_SPINLOCK <compileflags>-std=c++11 <compileflags>-stdlib=libc++ <compileflags>-miphoneos-version-min=7.0 <compileflags>-fvisibility=hidden <compileflags>-fvisibility-inlines-hidden <compileflags>-O2 <compileflags>-g <compileflags>-Os <compileflags>-Wno-long-long <compileflags>-Wno-missing-field-initializers <compileflags>-Wno-unused-variable <linkflags>-stdlib=libc++ <architecture>x86 <address-model>64 <target-os>iphone ; EOF


(2) patch boost_1_62_0/tools/build/src/tools/darwin.jam
~/Downloads/boost_1_62_0/tools/build/src/tools/darwin.jam    Select all
#change from case arm : { if $(instruction-set) { options = -arch$(_)$(instruction-set) ; } else { options = -arch arm ; } } #to case arm : { if $(instruction-set) { options = -arch$(_)$(instruction-set) ; } else if $(address-model) = 64 { options = -arch arm64 ; } else { options = -arch arm ; } }


(3) build boost libraries for iOS
shell script    Select all
# bootstrap cd $HOME/Downloads/boost_1_62_0 ./bootstrap.sh --with-libraries=atomic,chrono,date_time,exception,filesystem,graph,iostreams,math,program_options,random,regex,serialization,signals,system,test,thread,wave # clean and build for iOS, j option is num of cores x 1.5 rm -fr ios-build ./b2 -j6 --build-dir=ios-build --stagedir=ios-build/armv7 toolset=darwin-armv7 architecture=arm instruction-set=armv7 address-model=32 target-os=iphone threading=multi link=static stage ./b2 -j6 --build-dir=ios-build --stagedir=ios-build/armv7s toolset=darwin-armv7s architecture=arm instruction-set=armv7s address-model=32 target-os=iphone threading=multi link=static stage ./b2 -j6 --build-dir=ios-build --stagedir=ios-build/arm64 toolset=darwin-arm64 architecture=arm address-model=64 target-os=iphone threading=multi link=static stage ./b2 -j6 --build-dir=ios-build --stagedir=ios-build/i386 toolset=darwin-i386 architecture=x86 address-model=32 target-os=iphone threading=multi link=static stage ./b2 -j6 --build-dir=ios-build --stagedir=ios-build/x86_64 toolset=darwin-x86_64 architecture=x86 address-model=64 target-os=iphone threading=multi link=static stage # create libboost.a archive for each architecture cd $HOME/Downloads/boost_1_62_0 cd ios-build xcrun --sdk iphoneos ar crus armv7/libboost.a boost/bin.v2/libs/*/build/darwin-armv7/release/instruction-set-armv7/link-static/target-os-iphone/threading-multi/*/*.o xcrun --sdk iphoneos ar crus armv7s/libboost.a boost/bin.v2/libs/*/build/darwin-armv7s/release/instruction-set-armv7s/link-static/target-os-iphone/threading-multi/*/*.o xcrun --sdk iphoneos ar crus arm64/libboost.a boost/bin.v2/libs/*/build/darwin-arm64/release/link-static/target-os-iphone/threading-multi/*/*.o xcrun --sdk iphoneos ar crus i386/libboost.a boost/bin.v2/libs/*/build/darwin-i386/release/link-static/target-os-iphone/threading-multi/*/*.o xcrun --sdk iphoneos ar crus x86_64/libboost.a boost/bin.v2/libs/*/build/darwin-x86_64/release/link-static/target-os-iphone/threading-multi/*/*.o # create FAT libboost.a archive mkdir -p lib /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/lipo \ -arch armv7 "armv7/libboost.a" \ -arch armv7s "armv7s/libboost.a" \ -arch arm64 "arm64/libboost.a" \ -arch i386 "i386/libboost.a" \ -arch x86_64 "x86_64/libboost.a" \ -output "lib/libboost.a" \ -create # create link for include folder mkdir include cd include ln -s ../../boost .


(4) Create boost Framework for iOS
shell script    Select all
cd $HOME/Downloads/boost_1_62_0 cd ios-build #Create the following shell script for build-boostframework.sh cat > build-boostframework.sh << 'boostFrameworkEOF' #!/bin/bash # build-boostframework.sh VERSION_TYPE=Alpha FRAMEWORK_NAME=boost FRAMEWORK_VERSION=A FRAMEWORKDIR=. FRAMEWORK_CURRENT_VERSION=1.62.0 FRAMEWORK_COMPATIBILITY_VERSION=1.62.0 FRAMEWORK_BUNDLE=${FRAMEWORKDIR}/${FRAMEWORK_NAME}.framework rm -rf ${FRAMEWORK_BUNDLE} mkdir -p ${FRAMEWORK_BUNDLE} mkdir -p ${FRAMEWORK_BUNDLE}/Versions mkdir -p ${FRAMEWORK_BUNDLE}/Versions/${FRAMEWORK_VERSION} mkdir -p ${FRAMEWORK_BUNDLE}/Versions/${FRAMEWORK_VERSION}/Resources mkdir -p ${FRAMEWORK_BUNDLE}/Versions/${FRAMEWORK_VERSION}/Headers mkdir -p ${FRAMEWORK_BUNDLE}/Versions/${FRAMEWORK_VERSION}/Documentation ln -s $FRAMEWORK_VERSION ${FRAMEWORK_BUNDLE}/Versions/Current ln -s Versions/Current/Headers ${FRAMEWORK_BUNDLE}/Headers ln -s Versions/Current/Resources ${FRAMEWORK_BUNDLE}/Resources ln -s Versions/Current/Documentation ${FRAMEWORK_BUNDLE}/Documentation ln -s Versions/Current/$FRAMEWORK_NAME ${FRAMEWORK_BUNDLE}/$FRAMEWORK_NAME FRAMEWORK_INSTALL_NAME=${FRAMEWORK_BUNDLE}/Versions/${FRAMEWORK_VERSION}/${FRAMEWORK_NAME} echo "Framework: Lipoing library into ${FRAMEWORK_INSTALL_NAME}" /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/lipo \ -arch armv7 "armv7/libboost.a" \ -arch armv7s "armv7s/libboost.a" \ -arch arm64 "arm64/libboost.a" \ -arch i386 "i386/libboost.a" \ -arch x86_64 "x86_64/libboost.a" \ -output "${FRAMEWORK_INSTALL_NAME}" \ -create echo "Framework: Copying includes..." cp -r include/${FRAMEWORK_NAME}/* ${FRAMEWORK_BUNDLE}/Headers/ cat > ${FRAMEWORK_BUNDLE}/Resources/Info.plist <<InfoplistEOF <?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> <key>CFBundleDevelopmentRegion</key> <string>English</string> <key>CFBundleExecutable</key> <string>${FRAMEWORK_NAME}</string> <key>CFBundleIdentifier</key> <string>${FRAMEWORK_NAME}.org</string> <key>CFBundleInfoDictionaryVersion</key> <string>6.0</string> <key>CFBundlePackageType</key> <string>FMWK</string> <key>CFBundleSignature</key> <string>????</string> <key>CFBundleVersion</key> <string>${FRAMEWORK_CURRENT_VERSION}</string> </dict> </plist> InfoplistEOF echo "Done: ${FRAMEWORK_BUNDLE}" boostFrameworkEOF # build framework sh build-boostframework.sh


(5) Download and build QuantLib 1.9 for iOS
shell script    Select all
# download quantlib source cd $HOME/Downloads curl -O https://jaist.dl.sourceforge.net/project/quantlib/QuantLib/1.9/QuantLib-1.9.tar.gz tar xzvf QuantLib-1.9.tar.gz cd QuantLib-1.9 #Create the following shell script for armv7 cat > ~/Downloads/QuantLib-1.9/build-armv7.sh << 'armv7EOF' #!/bin/bash # build-armv7.sh BUILD_DIR=ios-build mkdir -p ${BUILD_DIR} INSTALL_DIR="`pwd`/${BUILD_DIR}" YOUR_PATH_TO_INSTALL_DIR="`cd "${INSTALL_DIR}";pwd`" XCODE_ROOT=/Applications/Xcode.app/Contents/Developer YOUR_TOOLCHAIN=${XCODE_ROOT}/Toolchains/XcodeDefault.xctoolchain/usr/bin sysroot=${XCODE_ROOT}/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk export CC=${YOUR_TOOLCHAIN}/clang export CXX=${YOUR_TOOLCHAIN}/clang++ export CPP=${YOUR_TOOLCHAIN}/cpp export AR=${YOUR_TOOLCHAIN}/ar export RANLIB=${YOUR_TOOLCHAIN}/ranlib export LD=${YOUR_TOOLCHAIN}/ld export CFLAGS="-arch armv7 -std=c99 -miphoneos-version-min=7.0 -isysroot ${sysroot}" export CXXFLAGS="-arch armv7 -fexceptions -frtti -fpic -std=c++11 -stdlib=libc++ -miphoneos-version-min=7.0 -O2 -g -fvisibility=hidden -fvisibility-inlines-hidden -Os -isysroot ${sysroot}" export LDFLAGS="-stdlib=libc++" boost_include="`cd "../boost_1_62_0/ios-build/include";pwd`" boost_lib="`cd "../boost_1_62_0/ios-build/armv7/lib";pwd`" make distclean ./configure --prefix=/armv7 --host=arm-apple-darwin10 --with-sysroot=${sysroot} --with-boost-include=${boost_include} --with-boost-lib=${boost_lib} --enable-static --enable-shared=no --disable-examples make -j6 make install DESTDIR=${YOUR_PATH_TO_INSTALL_DIR} armv7EOF #Create the following shell script for armv7s cat > ~/Downloads/QuantLib-1.9/build-armv7s.sh << 'armv7sEOF' #!/bin/bash # build-armv7s.sh BUILD_DIR=ios-build mkdir -p ${BUILD_DIR} INSTALL_DIR="`pwd`/${BUILD_DIR}" YOUR_PATH_TO_INSTALL_DIR="`cd "${INSTALL_DIR}";pwd`" XCODE_ROOT=/Applications/Xcode.app/Contents/Developer YOUR_TOOLCHAIN=${XCODE_ROOT}/Toolchains/XcodeDefault.xctoolchain/usr/bin sysroot=${XCODE_ROOT}/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk export CC=${YOUR_TOOLCHAIN}/clang export CXX=${YOUR_TOOLCHAIN}/clang++ export CPP=${YOUR_TOOLCHAIN}/cpp export AR=${YOUR_TOOLCHAIN}/ar export RANLIB=${YOUR_TOOLCHAIN}/ranlib export LD=${YOUR_TOOLCHAIN}/ld export CFLAGS="-arch armv7s -std=c99 -miphoneos-version-min=7.0 -isysroot ${sysroot}" export CXXFLAGS="-arch armv7s -fexceptions -frtti -fpic -std=c++11 -stdlib=libc++ -miphoneos-version-min=7.0 -O2 -g -fvisibility=hidden -fvisibility-inlines-hidden -Os -isysroot ${sysroot}" export LDFLAGS="-stdlib=libc++" boost_include="`cd "../boost_1_62_0/ios-build/include";pwd`" boost_lib="`cd "../boost_1_62_0/ios-build/armv7s/lib";pwd`" make distclean ./configure --prefix=/armv7s --host=arm-apple-darwin10 --with-sysroot=${sysroot} --with-boost-include=${boost_include} --with-boost-lib=${boost_lib} --enable-static --enable-shared=no --disable-examples make -j6 make install DESTDIR=${YOUR_PATH_TO_INSTALL_DIR} armv7sEOF #Create the following shell script for arm64 cat > ~/Downloads/QuantLib-1.9/build-arm64.sh << 'arm64EOF' #!/bin/bash # build-arm64.sh BUILD_DIR=ios-build mkdir -p ${BUILD_DIR} INSTALL_DIR="`pwd`/${BUILD_DIR}" YOUR_PATH_TO_INSTALL_DIR="`cd "${INSTALL_DIR}";pwd`" XCODE_ROOT=/Applications/Xcode.app/Contents/Developer YOUR_TOOLCHAIN=${XCODE_ROOT}/Toolchains/XcodeDefault.xctoolchain/usr/bin sysroot=${XCODE_ROOT}/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk export CC=${YOUR_TOOLCHAIN}/clang export CXX=${YOUR_TOOLCHAIN}/clang++ export CPP=${YOUR_TOOLCHAIN}/cpp export AR=${YOUR_TOOLCHAIN}/ar export RANLIB=${YOUR_TOOLCHAIN}/ranlib export LD=${YOUR_TOOLCHAIN}/ld export CFLAGS="-arch arm64 -std=c99 -miphoneos-version-min=7.0 -isysroot ${sysroot}" export CXXFLAGS="-arch arm64 -fexceptions -frtti -fpic -std=c++11 -stdlib=libc++ -miphoneos-version-min=7.0 -O2 -g -fvisibility=hidden -fvisibility-inlines-hidden -Os -isysroot ${sysroot}" export LDFLAGS="-stdlib=libc++" boost_include="`cd "../boost_1_62_0/ios-build/include";pwd`" boost_lib="`cd "../boost_1_62_0/ios-build/arm64/lib";pwd`" make distclean ./configure --prefix=/arm64 --host=arm-apple-darwin10 --with-sysroot=${sysroot} --with-boost-include=${boost_include} --with-boost-lib=${boost_lib} --enable-static --enable-shared=no --disable-examples make -j6 make install DESTDIR=${YOUR_PATH_TO_INSTALL_DIR} arm64EOF #Create the following shell script for i386 cat > ~/Downloads/QuantLib-1.9/build-i386.sh << 'i386EOF' #!/bin/bash # build-i386.sh BUILD_DIR=ios-build mkdir -p ${BUILD_DIR} INSTALL_DIR="`pwd`/${BUILD_DIR}" YOUR_PATH_TO_INSTALL_DIR="`cd "${INSTALL_DIR}";pwd`" XCODE_ROOT=/Applications/Xcode.app/Contents/Developer YOUR_TOOLCHAIN=${XCODE_ROOT}/Toolchains/XcodeDefault.xctoolchain/usr/bin sysroot=${XCODE_ROOT}/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk export CC=${YOUR_TOOLCHAIN}/clang export CXX=${YOUR_TOOLCHAIN}/clang++ export CPP=${YOUR_TOOLCHAIN}/cpp export AR=${YOUR_TOOLCHAIN}/ar export RANLIB=${YOUR_TOOLCHAIN}/ranlib export LD=${YOUR_TOOLCHAIN}/ld export CFLAGS="-arch i386 -std=c99 -miphoneos-version-min=7.0 -isysroot ${sysroot}" export CXXFLAGS="-arch i386 -fexceptions -frtti -fpic -std=c++11 -stdlib=libc++ -miphoneos-version-min=7.0 -O2 -g -fvisibility=hidden -fvisibility-inlines-hidden -Os -isysroot ${sysroot}" export LDFLAGS="-stdlib=libc++" boost_include="`cd "../boost_1_62_0/ios-build/include";pwd`" boost_lib="`cd "../boost_1_62_0/ios-build/i386/lib";pwd`" make distclean ./configure --prefix=/i386 --host=i386-apple-darwin10 --with-sysroot=${sysroot} --with-boost-include=${boost_include} --with-boost-lib=${boost_lib} --enable-static --enable-shared=no --disable-examples make -j6 make install DESTDIR=${YOUR_PATH_TO_INSTALL_DIR} i386EOF #Create the following shell script for x86_64 cat > ~/Downloads/QuantLib-1.9/build-x86_64.sh << 'x86_64EOF' #!/bin/bash # build-x86_64.sh BUILD_DIR=ios-build mkdir -p ${BUILD_DIR} INSTALL_DIR="`pwd`/${BUILD_DIR}" YOUR_PATH_TO_INSTALL_DIR="`cd "${INSTALL_DIR}";pwd`" XCODE_ROOT=/Applications/Xcode.app/Contents/Developer YOUR_TOOLCHAIN=${XCODE_ROOT}/Toolchains/XcodeDefault.xctoolchain/usr/bin sysroot=${XCODE_ROOT}/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk export CC=${YOUR_TOOLCHAIN}/clang export CXX=${YOUR_TOOLCHAIN}/clang++ export CPP=${YOUR_TOOLCHAIN}/cpp export AR=${YOUR_TOOLCHAIN}/ar export RANLIB=${YOUR_TOOLCHAIN}/ranlib export LD=${YOUR_TOOLCHAIN}/ld export CFLAGS="-arch x86_64 -std=c99 -miphoneos-version-min=7.0 -isysroot ${sysroot}" export CXXFLAGS="-arch x86_64 -fexceptions -frtti -fpic -std=c++11 -stdlib=libc++ -miphoneos-version-min=7.0 -O2 -g -fvisibility=hidden -fvisibility-inlines-hidden -Os -isysroot ${sysroot}" export LDFLAGS="-stdlib=libc++" boost_include="`cd "../boost_1_62_0/ios-build/include";pwd`" boost_lib="`cd "../boost_1_62_0/ios-build/x86_64/lib";pwd`" make distclean ./configure --prefix=/x86_64 --host=x86_64-apple-darwin10 --with-sysroot=${sysroot} --with-boost-include=${boost_include} --with-boost-lib=${boost_lib} --enable-static --enable-shared=no --disable-examples make -j6 make install DESTDIR=${YOUR_PATH_TO_INSTALL_DIR} x86_64EOF #run shell scripts to clean and build for iOS cd $HOME/Downloads cd QuantLib-1.9 rm -fr ios-build sh build-armv7.sh sh build-armv7s.sh sh build-arm64.sh sh build-i386.sh sh build-x86_64.sh # create FAT libQuantLib.a archive cd $HOME/Downloads/QuantLib-1.9 cd ios-build mkdir -p lib /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/lipo \ -arch armv7 "armv7/lib/libQuantLib.a" \ -arch armv7s "armv7s/lib/libQuantLib.a" \ -arch arm64 "arm64/lib/libQuantLib.a" \ -arch i386 "i386/lib/libQuantLib.a" \ -arch x86_64 "x86_64/lib/libQuantLib.a" \ -output "lib/libQuantLib.a" \ -create # create link for include folder mkdir include cd include ln -s ../arm64/include/ql .


(6) Create QuantLib Framework for iOS
shell script    Select all
cd $HOME/Downloads/QuantLib-1.9 cd ios-build #Create the following shell script for build-qlframework.sh cat > build-qlframework.sh << 'qlFrameworkEOF' #!/bin/bash # build-qlframework.sh VERSION_TYPE=Alpha FRAMEWORK_NAME=ql FRAMEWORK_VERSION=A FRAMEWORKDIR=. FRAMEWORK_CURRENT_VERSION=1.9 FRAMEWORK_COMPATIBILITY_VERSION=1.9 FRAMEWORK_BUNDLE=${FRAMEWORKDIR}/${FRAMEWORK_NAME}.framework rm -rf ${FRAMEWORK_BUNDLE} mkdir -p ${FRAMEWORK_BUNDLE} mkdir -p ${FRAMEWORK_BUNDLE}/Versions mkdir -p ${FRAMEWORK_BUNDLE}/Versions/${FRAMEWORK_VERSION} mkdir -p ${FRAMEWORK_BUNDLE}/Versions/${FRAMEWORK_VERSION}/Resources mkdir -p ${FRAMEWORK_BUNDLE}/Versions/${FRAMEWORK_VERSION}/Headers mkdir -p ${FRAMEWORK_BUNDLE}/Versions/${FRAMEWORK_VERSION}/Documentation ln -s $FRAMEWORK_VERSION ${FRAMEWORK_BUNDLE}/Versions/Current ln -s Versions/Current/Headers ${FRAMEWORK_BUNDLE}/Headers ln -s Versions/Current/Resources ${FRAMEWORK_BUNDLE}/Resources ln -s Versions/Current/Documentation ${FRAMEWORK_BUNDLE}/Documentation ln -s Versions/Current/$FRAMEWORK_NAME ${FRAMEWORK_BUNDLE}/$FRAMEWORK_NAME FRAMEWORK_INSTALL_NAME=${FRAMEWORK_BUNDLE}/Versions/${FRAMEWORK_VERSION}/${FRAMEWORK_NAME} echo "Framework: Lipoing library into ${FRAMEWORK_INSTALL_NAME}" /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/lipo \ -arch armv7 "armv7/lib/libQuantLib.a" \ -arch armv7s "armv7s/lib/libQuantLib.a" \ -arch arm64 "arm64/lib/libQuantLib.a" \ -arch i386 "i386/lib/libQuantLib.a" \ -arch x86_64 "x86_64/lib/libQuantLib.a" \ -output "${FRAMEWORK_INSTALL_NAME}" \ -create echo "Framework: Copying includes..." cp -r include/${FRAMEWORK_NAME}/* ${FRAMEWORK_BUNDLE}/Headers/ cat > ${FRAMEWORK_BUNDLE}/Resources/Info.plist <<InfoplistEOF <?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> <key>CFBundleDevelopmentRegion</key> <string>English</string> <key>CFBundleExecutable</key> <string>${FRAMEWORK_NAME}</string> <key>CFBundleIdentifier</key> <string>quantlib.org</string> <key>CFBundleInfoDictionaryVersion</key> <string>6.0</string> <key>CFBundlePackageType</key> <string>FMWK</string> <key>CFBundleSignature</key> <string>????</string> <key>CFBundleVersion</key> <string>${FRAMEWORK_CURRENT_VERSION}</string> </dict> </plist> InfoplistEOF echo "Done: ${FRAMEWORK_BUNDLE}" qlFrameworkEOF # build framework sh build-qlframework.sh



Sunday, September 11, 2016

How to split CSV in Swift

Below is Swift 2.1 code
Swift 3 code is here https://github.com/Daniel1of1/CSwiftV/blob/master/Sources/CSwiftV.swift
csvsplit.swift   Select all
//: Playground - noun: a place where people can play import Foundation public func split(string: String, _ separator: String = ",") -> [String] { // Swift 2.1 //public func split(_ string: String, _ separator: String = ",") -> [String] { // Swift 3 func oddNumberOfQuotes(string: String) -> Bool { // Swift 2.1 // func oddNumberOfQuotes(_ string: String) -> Bool { // Swift 3 return string.componentsSeparatedByString("\"").count % 2 == 0 // Swift 2.1 // return string.components(separatedBy: "\"").count % 2 == 0 // Swift 3 } let initial = string.componentsSeparatedByString(separator) // Swift 2.1 // let initial = string.components(separatedBy: separator) // Swift 3 var merged = [String]() for newString in initial { if let record = merged.last { guard oddNumberOfQuotes(record) == true else { merged.append(newString) continue } merged.removeLast() let lastElem = record + separator + newString merged.append(lastElem) } else { merged.append(newString) continue } } return merged } public func csvsplit(string: String, _ rowSeparator: String = "\n", _ colSeparator: String = ",") -> [[String]] { // Swift 2.1 //public func csvsplit(_ string: String, _ rowSeparator: String = "\n", _ colSeparator: String = ",") -> [[String]] { // Swift 3 return split(string.stringByReplacingOccurrencesOfString("\r\n", withString: "\n"), rowSeparator).map{split($0, colSeparator)} // Swift 2.1 // return split(string.replacingOccurrences(of: "\r\n", with: "\n"), rowSeparator).map{split($0, colSeparator)} // Swift 3 } print(csvsplit("aa,\"b,b\",cc\na,b,c")) do { // www.quandl.com api : anonymous user limit of 50 calls per day. let csv = try String(contentsOfURL: NSURL(string: "https://www.quandl.com/api/v3/datasets/FRED/DEXJPUS.csv")!, encoding: NSUTF8StringEncoding) // Swift 2.1 // let csv = try String(contentsOf: Foundation.URL(string: "https://www.quandl.com/api/v3/datasets/FRED/DEXJPUS.csv")!) // Swift 3 // there are over 11 000 rows from this url, so playground will take some times to finish print(split(csv,"\n")[0...2].map{split($0)}) } catch let error { print(error) }



Thursday, September 1, 2016

Struct and Enum used together in Swift 3

Sample code in swift 3 (Xcode 8 beta 6)

This demonstrates
(1) the use of enum initialization method for default value
(2) the use of struct initialization method for default variable value
(3) the use of multiple guard statements in Swift 3
(4) struct and enum used together for data model that has different associated values based on enum type
(5) internal var for mutating associated values
playgroundcode.swift   Select all
//: Playground - noun: a place where people can play import Foundation struct Instrument { enum CurrencyType:String { case USD case EUR case GBP case JPY case EURUSD case USDEUR case GBPUSD case USDGBP case USDJPY case JPYUSD case USDCNY case USDHKD // default initialization methods for currencyType enum init() { self = .USD } } enum InstrumentType { case FloatingRateBond(ccy:String, faceamount: Double?, redemption: Double?, issueDate:String, maturityDate:String, floatingRate:String) case FixedRateBond(ccy:String, faceamount: Double?, redemption: Double?, issueDate:String, maturityDate:String, fixedRate:String) case ZeroCouponBond(ccy:String, faceamount: Double?, redemption: Double?, issueDate:String, maturityDate:String) case Unknown // default initialization methods for different instrumentType enum init() { self = .Unknown } init(ccy: String, issueDate:String, maturityDate:String, floatingRate:String) { self = .FloatingRateBond(ccy: ccy, faceamount: 100.0, redemption: 100.0, issueDate: issueDate, maturityDate: maturityDate, floatingRate: floatingRate) } init(ccy: String, issueDate:String, maturityDate:String, fixedRate:String) { self = .FixedRateBond(ccy: ccy, faceamount: 100.0, redemption: 100.0, issueDate: issueDate, maturityDate: maturityDate, fixedRate: fixedRate) } init(ccy: String, redemption:Double?, issueDate:String, maturityDate:String) { self = .ZeroCouponBond(ccy: ccy, faceamount: 100.0, redemption: redemption, issueDate: issueDate, maturityDate: maturityDate) } } var title:String? = nil var name:String var type:InstrumentType var typeName: String { switch self.type { case .FloatingRateBond : return "FloatingRateBond" case .FixedRateBond : return "FixedRateBond" case .ZeroCouponBond : return "ZeroCouponBond" default : return "Unknown" } } // provide mutating internal var internal var _ccy : String? = nil var ccy : String? { get { guard let ccy: String = { switch self.type { // use case let here case let .FloatingRateBond(tuple) : return self._ccy ?? tuple.ccy case let .FixedRateBond(tuple) : return self._ccy ?? tuple.ccy case let .ZeroCouponBond(tuple) : return self._ccy ?? tuple.ccy default : return nil } }() else { return nil } return ccy } set { // cannot throws error here yet switch self.type { case .FloatingRateBond, .FixedRateBond, .ZeroCouponBond : _ccy = newValue! default : () } } } var currencyType : CurrencyType? { // multiple guard statements Swift 3 guard let ccy = self.ccy, let ccytype:CurrencyType = CurrencyType(rawValue: ccy.uppercased()) else { return nil } return ccytype } // no mutating internal var var faceAmount : Double? { guard let faceAmount: Double = { switch self.type { // use pattern matching here case .FloatingRateBond(_, let faceAmount, _, _, _, _) : return faceAmount case .FixedRateBond(_, let faceAmount, _, _, _, _) : return faceAmount case .ZeroCouponBond(_, let faceAmount, _, _, _) : return faceAmount default : return nil } }() else { return nil } return faceAmount } var redemption : Double? { guard let redemption: Double = { switch self.type { case .FloatingRateBond(_, _, let redemption, _, _, _) : return redemption case .FixedRateBond(_, _, let redemption, _, _, _) : return redemption case .ZeroCouponBond(_, _, let redemption, _, _) : return redemption default : return nil } }() else { return nil } return redemption } var issueDate : String? { guard let issueDate: String = { switch self.type { case .FloatingRateBond(_, _, _, let issueDate, _, _) : return issueDate case .FixedRateBond(_, _, _, let issueDate, _, _) : return issueDate case .ZeroCouponBond(_, _, _, let issueDate, _) : return issueDate default : return nil } }() else { return nil } return issueDate.isEmpty ? nil : issueDate } var maturityDate : String? { guard let maturityDate: String = { switch self.type { case .FloatingRateBond(_, _, _, _, let maturityDate, _) : return maturityDate case .FixedRateBond(_, _, _, _, let maturityDate, _) : return maturityDate case .ZeroCouponBond(_, _, _, _, let maturityDate) : return maturityDate default : return nil } }() else { return nil } return maturityDate.isEmpty ? nil : maturityDate } var description: String { switch self.type { case .FloatingRateBond(let ccy, _, _, let issueDate, let maturityDate, let floatingRate) : return "\(self.name) FloatingRateBond \(ccy) \(issueDate) \(maturityDate) \(floatingRate)" case .FixedRateBond(let ccy, _, _, let issueDate, let maturityDate, let fixedRate) : return "\(self.name) FixedRateBond \(ccy) \(issueDate) \(maturityDate) \(fixedRate)" case .ZeroCouponBond : return "ZeroCouponBond" default : return "Unknown Description" } } var marketData:AnyObject? = nil var pricingEngine:AnyObject? = nil // default initialization methods for Instrument struct init(name: String, type: InstrumentType) { self.name = name self.type = type } init(name: String) { self.name = name self.type = InstrumentType() } func NPV() -> Double? { // multiple guard statements Swift 3 guard let data = self.marketData, let engine = self.pricingEngine else { return nil } // TODO: marketData and pricingEngine print("\(data) \(engine)") return 0.0 } } var portfolio = [ Instrument(name:"B00001USDLIBOR", type: Instrument.InstrumentType(ccy: "USD", issueDate: "20100101", maturityDate: "20101231", floatingRate: "LIBOR+1")), Instrument(name:"B0002JPY0.5%", type: .FixedRateBond(ccy: "JPY", faceamount: 100.0, redemption: 100.0, issueDate: "20110101", maturityDate: "20291231", fixedRate: "0.5%")), Instrument(name:"ZC0001", type: Instrument.InstrumentType(ccy: "USD", redemption: 120.0, issueDate: "20110101", maturityDate: "20291231")), Instrument(name:"U0001")] portfolio.forEach{ ( a : Instrument) -> () in print(a.typeName) print(a) if let b = a.currencyType { print(b) } else { print("missing / invalid currency info \(a.ccy)") } print(a.issueDate ?? "No issueDate") print(a.maturityDate ?? "No maturityDate") print(a.description) print(a.NPV() ?? "No NPV") } print(portfolio[2].ccy) portfolio[2].ccy = "JPY" print(portfolio[2].currencyType) print(portfolio[2].ccy)



Differences between struct Instrument and class Instrument
(1) class is reference type and struct is value type.
(2) open class can be subclassed in other modules, but not for struct.
(3) class func is not allowed, use static func instead in struct.
(4) reference and value types can be modified in the forEach or other iteration methods, as below.
(5) modification inside iteration is local copy only for struct due to value type nature.
playgroundcode.swift   Select all
open class Instrument { .... } .... // cannot set var a here in function parameter for Swift 3 (SE-0003) portfolio.forEach{ ( a : Instrument) -> () in var a = a // using shadow variable switch a.type { case .Unknown : a.type = Instrument.InstrumentType(ccy: "GBP", redemption: 110.0, issueDate: "20010101", maturityDate: "20291231") // modification is local copy only inside iteration for struct a.name = "MODIFIED NAME" default: () // do nothing } print(a.typeName) print(a) if let t = a.currencyType { print(t) } else { print("missing / invalid currency info \(a.ccy)") } print(a.issueDate ?? "No issueDate") print(a.maturityDate ?? "No maturityDate") print(a.description) print(a.NPV() ?? "No NPV") } print(portfolio[3].name) // instance can be modified inside print(portfolio[3].type) // modification is local copy only inside iteration for struct



Tuesday, August 30, 2016

How to parse JSON using JSONSerialization in Swift 3

Sample code in swift 3 (Xcode 8 beta 6) with Swift 2 code shown beneath
samplecode.swift   Select all
import Foundation let str1 = "{\"names\": [\"Bob\", \"Tim\", \"Tina\"]}" let str2 = "{\"names\": [\"Bob2\", \"Tim2\", \"Tina2\"]};" for str in [str1, str2] { let data = str.data(using: String.Encoding.utf8) // let data = str.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: false) // Swift 2 do { let json = try JSONSerialization.jsonObject(with: data!) as! [String: AnyObject] // let json = try NSJSONSerialization.JSONObjectWithData(data!, options: []) // Swift 2 print(json) if let names = json["names"] as? [String] { print(names) } } catch let error as NSError { print(str) print("Failed to load: \(error.localizedDescription)") } }



Sample Code with json from url in swift 3 (Xcode 8 beta 6) with Swift 2 code shown beneath
samplecode.swift   Select all
import Foundation struct Video { var name: String var url: String } var musicvideo:[Video]=[] typealias JSONDictionary = [String: AnyObject] typealias JSONArray = Array<AnyObject> let url = Foundation.URL(string: "https://itunes.apple.com/us/rss/topmusicvideos/limit=3/json") //let url = NSURL(string: "https://itunes.apple.com/us/rss/topmusicvideos/limit=3/json") // Swift 2 do { let data = try Data(contentsOf: url!) // let data = NSData(contentsOfURL: url!) // Swift 2 let json = try JSONSerialization.jsonObject(with: data, options: .allowFragments) as! JSONDictionary // .allowFragments - top level object is not Array nor Dictionary. // let json = try NSJSONSerialization.JSONObjectWithData(data!, options: .AllowFragments) // Swift 2 if let feed = json["feed"] as? JSONDictionary, let entries = feed["entry"] as? JSONArray { // entries = feed["entry"] as? JSONArray { // Swift 2 //print(entries) for (index, entry) in entries.enumerated() { // for (index, entry) in entries.enumerate() { // Swift 2 //print("\(index) is \(entry)") if let name = entry["im:name"] as? JSONDictionary, let vName = name["label"] as? String, // SE-0099 XCode 8 beta 4 // vName = name["label"] as? String, // Swift 2 let video = entry["link"] as? JSONArray, // SE-0099 XCode 8 beta 4 // video = entry["link"] as? JSONArray, // Swift 2 let vUrl = video[1] as? JSONDictionary, // SE-0099 XCode 8 beta 4 // vUrl = video[1] as? JSONDictionary, // Swift 2 let vHref = vUrl["attributes"] as? JSONDictionary, // SE-0099 XCode 8 beta 4 // vHref = vUrl["attributes"] as? JSONDictionary, // Swift 2 let vVideoUrl = vHref["href"] as? String // SE-0099 XCode 8 beta 4 // vVideoUrl = vHref["href"] as? String // Swift 2 { print(vName) print(vVideoUrl) musicvideo.append(Video(name: vName, url:vVideoUrl)) } } } } catch let error as NSError { print("Failed to load: \(error.localizedDescription)") } let jsonDict = musicvideo.map { return ["name":$0.name, "url":$0.url]} print(jsonDict)



Thursday, July 28, 2016

How to write custom XMLParser for Swift 3

There are some general purpose XMLParsers libraries on github e.g. SWXMLHash or XML

Sometimes, it is faster to write your own and can use it in Playgrounds. What you need for a simple XMLParser is to define a class with XMLParserDelegate and write parsing rules in "didStartElement", "foundCharacters" and "didEndElement" methods in the delegated class.

Below is an example code in swift 3 (Xcode 8 beta 6) and with Swift 2.1 code added as comments underneath. And try to parse the xml file from quandl.com online dataset.
// curl "https://www.quandl.com/api/v3/datasets/FRED/DEXJPUS.xml"

These codes do not work on Linux Swift due to incomplete implementation of Foundation library on Linux yet.

testQuandl_main.swift    Select all
// // main.swift // testQuandl // import Foundation func timetest(_ note: String, block: () -> Void) //func timetest(note: String, block: () -> Void) // Swift 2 { let date = Date() // let date = NSDate() // Swift 2 block() let timeInterval = Date().timeIntervalSince(date) // let timeInterval = NSDate().timeIntervalSinceDate(date) // Swift 2 print("Test:", note); print("Elapsed time: \(String(format: "%.2f", timeInterval)) s") } extension String { func trimmed() -> String { return self.trimmingCharacters(in: .whitespacesAndNewlines) //return self.trimmingCharacters(in: NSCharacterSet.whitespacesAndNewlines()) // Xcode 8 beta 1 //return self.stringByTrimmingCharactersInSet(NSCharacterSet.whitespaceAndNewlineCharacterSet()) // Swift 2 } } class QuandlCCYParser : NSObject, Foundation.XMLParserDelegate { // class QuandlCCYParser : NSObject, Foundation.NSXMLParserDelegate { // Swift 2 var parser = Foundation.XMLParser() // var parser = Foundation.NSXMLParser() // Swift 2 fileprivate var element:String = String() fileprivate var type:String = String() fileprivate var elements:Dictionary<String,String>? = [String: String]() var dataset = [Dictionary<String,String>]() fileprivate var name:String? = String() fileprivate var newest_available_date:String? = String() var dataPoints:[Dictionary<String,String>] = [[String:String]]() fileprivate var date:String? = String() fileprivate var value:String? = String() func startParsing(_ xmltxt:String) { // func startParsing(xmltxt:String) { // Swift 2 dataset = [] let xmlData = xmltxt.data(using: String.Encoding.utf8)! // let xmlData = xmltxt.dataUsingEncoding(NSUTF8StringEncoding)! // Swift 2 parser = Foundation.XMLParser(data: xmlData) // parser = Foundation.NSXMLParser(data: xmlData) // Swift 2 parser.delegate = self parser.parse() } func startParsing(urlpath:String) { // func startParsing(urlpath urlpath:String) { // Swift 2 dataset = [] parser = Foundation.XMLParser(contentsOf:(Foundation.URL(string:urlpath))! as Foundation.URL)! // parser = Foundation.NSXMLParser(contentsOfURL:(Foundation.NSURL(string:urlpath))! as Foundation.NSURL)! // Swift 2 parser.delegate = self parser.parse() } func parser(_ parser: Foundation.XMLParser, // func parser(parser: Foundation.NSXMLParser, // Swift 2 didStartElement elementName: String, namespaceURI: String?, qualifiedName qName: String?, attributes attributeDict: [String: String]) { element = elementName if let type = attributeDict["type"] { self.type = type } else { self.type = "" } if (elementName == "datum" && attributeDict["type"] == "date") { date = nil elements = nil } else if (elementName == "datum" && attributeDict["type"] == "float") { value = nil elements = nil } else if (elementName == "name") { elements = nil name = nil } else if (elementName == "newest-available-date" && attributeDict["type"] == "date") { elements = nil newest_available_date = nil } } func parser(_ parser: Foundation.XMLParser, // func parser(parser: Foundation.NSXMLParser, // Swift 2 foundCharacters string: String) { if element == "datum" && type == "date" { if date == nil { date = "" } date!.append(string) // date! += string // Swift 2 } else if element == "datum" && type == "float" { if value == nil { value = "" } value!.append(string) // value! += string // Swift 2 } else if element == "name" { if name == nil { name = "" } name!.append(string) // name! += string // Swift 2 } else if element == "newest-available-date" && type == "date" { if newest_available_date == nil { newest_available_date = "" } newest_available_date!.append(string) // newest_available_date! += string // Swift 2 } } func parser(_ parser: Foundation.XMLParser, // func parser(parser: Foundation.NSXMLParser, // Swift 2 didEndElement elementName: String, namespaceURI: String?, qualifiedName qName: String?) { if (elementName == "datum" && type == "float") { if (date != nil && value != nil) { elements = [:] elements!["date"] = date!.trimmed() elements!["value"] = value!.trimmed() } if elements != nil { dataPoints.append(elements!) (date, value, elements) = (nil, nil, nil) } } else if elementName == "dataset" { elements = [:] if name != nil { elements!["name"] = name!.trimmed() } if newest_available_date != nil { elements!["newest-available-date"] = newest_available_date!.trimmed() } if elements != nil { dataset.append(elements!) } (date, value, elements) = (nil, nil, nil) } } func showdataset() { for data in dataset { if let name = data["name"], let newest_available_date = data["newest-available-date"] { // SE-0099 XCode 8 beta 4 // if let name = data["name"], newest_available_date = data["newest-available-date"] { // Swift 2 print("\(name) \(newest_available_date)") } } } } func test1() { let test = QuandlCCYParser() // curl "https://www.quandl.com/api/v3/datasets/FRED/DEXJPUS.xml" test.startParsing(urlpath:"https://www.quandl.com/api/v3/datasets/FRED/DEXJPUS.xml") test.showdataset() for data in test.dataPoints.reversed() where data["date"]! >= "2016-01-01" { // for data in test.dataPoints.reverse() where data["date"] >= "2016-01-01" { // Swift 2 if let date = data["date"], let value = Float(data["value"]!) { // SE-0099 XCode 8 beta 4 // if let date = data["date"], value = Float(data["value"]!) { // Swift 2 print("Date \(date) , USDJPY = \(String(format: "%.2f", value)) JPYUSD = \(1/value)") } } } timetest("DEXJPUS XMLParser", block:test1)

Xcode 8 Beta output




Below is another example XMLParser for Markit Interest Rate Curve Data and to retrieve curvepoints and other field items under Deposits and Swaps tags.
testMarkit_main.swift    Select all
// // main.swift // testMarkit // import Foundation func timetest(_ note: String, block: () -> Void) //func timetest(note: String, block: () -> Void) // Swift 2 { let date = Date() // let date = NSDate() // Swift 2 block() let timeInterval = Date().timeIntervalSince(date) // let timeInterval = NSDate().timeIntervalSinceDate(date) // Swift 2 print("Test:", note); print("Elapsed time: \(String(format: "%.2f", timeInterval)) s") } extension String { func trimmed() -> String { return self.trimmingCharacters(in: .whitespacesAndNewlines) //return self.trimmingCharacters(in: NSCharacterSet.whitespacesAndNewlines()) // Xcode 8 beta 1 //return self.stringByTrimmingCharactersInSet(NSCharacterSet.whitespaceAndNewlineCharacterSet()) // Swift 2 } } class ParserBase : NSObject, Foundation.XMLParserDelegate { // class ParserBase : NSObject, Foundation.NSXMLParserDelegate { // Swift 2 var parser = Foundation.XMLParser() // var parser = Foundation.NSXMLParser() // Swift 2 var currentElement:String = "" var foundCharacters = "" weak var parent:ParserBase? = nil func startParsing(_ xmltxt:String) { // func startParsing(xmltxt:String) { // Swift 2 let xmlData = xmltxt.data(using: String.Encoding.utf8)! // let xmlData = xmltxt.dataUsingEncoding(NSUTF8StringEncoding) // Swift 2 parser = Foundation.XMLParser(data: xmlData) // parser = Foundation.NSXMLParser(data: xmlData) // Swift 2 parser.delegate = self parser.parse() } func startParsing(urlpath:String) { // func startParsing(urlpath urlpath:String) { // Swift 2 parser = Foundation.XMLParser(contentsOf:(Foundation.URL(string:urlpath))! as Foundation.URL)! // parser = Foundation.NSXMLParser(contentsOfURL:(Foundation.NSURL(string:urlpath))! as Foundation.NSURL)! // Swift 2 parser.delegate = self parser.parse() } func startParsing(url:URL) { // func startParsing(url url:Foundation.NSURL) { // Swift 2 parser = Foundation.XMLParser(contentsOf:url)! // parser = Foundation.NSXMLParser(contentsOfURL:url)! // Swift 2 parser.delegate = self parser.parse() } func parser(_ parser: Foundation.XMLParser, // func parser(parser: Foundation.NSXMLParser, // Swift 2 didStartElement elementName: String, namespaceURI: String?, qualifiedName qName: String?, attributes attributeDict: [String: String]) { currentElement = elementName } func parser(_ parser: Foundation.XMLParser, foundCharacters string: String) // func parser(parser: Foundation.NSXMLParser, foundCharacters string: String) // Swift 2 { self.foundCharacters += string } } class InterestRateCurve : ParserBase { var currency = "" var effectiveasof = "" lazy var deposits = Deposit() lazy var swaps = Swap() override func parser(_ parser: Foundation.XMLParser, // override func parser(parser: Foundation.NSXMLParser, // Swift 2 didStartElement elementName: String, namespaceURI: String?, qualifiedName qName: String?, attributes attributeDict: [String: String]) { // if we found a tag, delegate further responsibility // to parsing to a new instance of Deposit or Swap if elementName == "deposits" { self.deposits = Deposit() // push responsibility parser.delegate = deposits // let delegate know who is the parent // so that once done XML processing // it can return parsing responsibility back deposits.parent = self } else if elementName == "swaps" { self.swaps = Swap() // push responsibility parser.delegate = swaps // let delegate know who is the parent // so that once done XML processing // it can return parsing responsibility back swaps.parent = self } super.parser(parser, didStartElement: elementName, namespaceURI: namespaceURI, qualifiedName: qName, attributes: attributeDict) } func parser(_ parser: Foundation.XMLParser, didEndElement elementName: String, namespaceURI: String?, qualifiedName qName: String?) // func parser(parser: Foundation.NSXMLParser, didEndElement elementName: String, namespaceURI: String?, qualifiedName qName: String?) // Swift 2 { // get other elements for deposits if elementName == "currency" { self.currency = foundCharacters.trimmed() } else if elementName == "effectiveasof" { self.effectiveasof = foundCharacters.trimmed() } // reset found characters foundCharacters = "" } } class Deposit : ParserBase { var curvepoints = [CurvePoint]() var spotdate:String = "" var daycountconvention:String = "" var snaptime:String = "" override func parser(_ parser: Foundation.XMLParser, // override func parser(parser: Foundation.NSXMLParser, // Swift 2 didStartElement elementName: String, namespaceURI: String?, qualifiedName qName: String?, attributes attributeDict: [String: String]) { if elementName == "curvepoint" { let depo = CurvePoint() self.curvepoints.append(depo) // push responsibility parser.delegate = depo // let delegate know who is the parent depo.parent = self } super.parser(parser, didStartElement: elementName, namespaceURI: namespaceURI, qualifiedName: qName, attributes: attributeDict) } func parser(_ parser: Foundation.XMLParser, didEndElement elementName: String, namespaceURI: String?, qualifiedName qName: String?) // func parser(parser: Foundation.NSXMLParser, didEndElement elementName: String, namespaceURI: String?, qualifiedName qName: String?) // Swift 2 { // print("processing didEndElement <\(elementName)> tag from Deposit") // if we reached the tag, we do not // have anything further to do, so delegate // parsing responsibility to parent if elementName == "deposits" { parser.delegate = self.parent } // get other field items for deposits else if elementName == "spotdate" { self.spotdate = foundCharacters.trimmed() } else if elementName == "daycountconvention" { self.daycountconvention = foundCharacters.trimmed() } else if elementName == "snaptime" { self.snaptime = foundCharacters.trimmed() } // reset found characters foundCharacters = "" } } class Swap : ParserBase { var curvepoints = [CurvePoint]() var spotdate:String = "" var fixeddaycountconvention:String = "" var floatingdaycountconvention:String = "" var fixedpaymentfrequency:String = "" var floatingpaymentfrequency:String = "" var snaptime:String = "" override func parser(_ parser: Foundation.XMLParser, // override func parser(parser: Foundation.NSXMLParser, // Swift 2 didStartElement elementName: String, namespaceURI: String?, qualifiedName qName: String?, attributes attributeDict: [String: String]) { if elementName == "curvepoint" { let swap = CurvePoint() self.curvepoints.append(swap) // push responsibility parser.delegate = swap // let delegate know who is the parent // so that it can return parsing responsibility // back after XML processing is done swap.parent = self } super.parser(parser, didStartElement: elementName, namespaceURI: namespaceURI, qualifiedName: qName, attributes: attributeDict) } func parser(_ parser: Foundation.XMLParser, didEndElement elementName: String, namespaceURI: String?, qualifiedName qName: String?) // func parser(parser: Foundation.NSXMLParser, didEndElement elementName: String, namespaceURI: String?, qualifiedName qName: String?) // Swift 2 { // print("processing didEndElement <\(elementName)> tag from Swap") // if we reached the </swaps> tag, we do not // have anything further to do, so delegate // parsing responsibility to parent if elementName == "swaps" { parser.delegate = self.parent } // get other field items for swaps else if elementName == "spotdate" { self.spotdate = foundCharacters.trimmed() } else if elementName == "fixeddaycountconvention" { self.fixeddaycountconvention = foundCharacters.trimmed() } else if elementName == "floatingdaycountconvention" { self.floatingdaycountconvention = foundCharacters.trimmed() } else if elementName == "fixedpaymentfrequency" { self.fixedpaymentfrequency = foundCharacters.trimmed() } else if elementName == "floatingpaymentfrequency" { self.floatingpaymentfrequency = foundCharacters.trimmed() } else if elementName == "snaptime" { self.snaptime = foundCharacters.trimmed() } // reset found characters foundCharacters = "" } } class CurvePoint : ParserBase { var tenor = "" var maturitydate = "" var parrate:Float = 0.0 func parser(_ parser: Foundation.XMLParser, didEndElement elementName: String, namespaceURI: String?, qualifiedName qName: String?) // func parser(parser: Foundation.NSXMLParser, didEndElement elementName: String, namespaceURI: String?, qualifiedName qName: String?) // Swift 2 { // print("processing didEndElement <\(elementName)> tag from CurvePoint") // if we finished an tag, the ParserBase parent // would have accumulated the found characters // so just assign that to our variable if elementName == "tenor" { self.tenor = foundCharacters.trimmed() } else if elementName == "maturitydate" { self.maturitydate = foundCharacters.trimmed() } // similarly for other tags else if elementName == "parrate" { if let l = Float(foundCharacters.trimmed()) { self.parrate = l } } // if we reached the </curvepoint> tag, we do not // have anything further to do, so delegate // parsing responsibility to parent else if elementName == "curvepoint" { parser.delegate = self.parent } // reset found characters foundCharacters = "" } } func printcurve(_ xmltxt:String) -> Void { //func printcurve(xmltxt:String) -> Void { // Swift 2 let interestRateCurve = InterestRateCurve() interestRateCurve.startParsing(xmltxt) print("interestRateCurve currency is \(interestRateCurve.currency) and effectiveasof is \(interestRateCurve.effectiveasof)") print(String(repeating: "-", count: 65)) // SE-0130 Xcode 8 beta 6 // print(String(repeating: ("-" as Character), count: 65)) // Xcode 8 beta 1 print("Deposits spotdate is \(interestRateCurve.deposits.spotdate) daycountconvention is \(interestRateCurve.deposits.daycountconvention)") for depo in interestRateCurve.deposits.curvepoints { print("Depo tenor = \(depo.tenor) maturitydate = \(depo.maturitydate) and parrate = \(depo.parrate)") } print("") print("Swaps spotdate is \(interestRateCurve.swaps.spotdate) fixeddaycountconvention is \(interestRateCurve.swaps.fixeddaycountconvention) floatingdaycountconvention is \(interestRateCurve.swaps.floatingdaycountconvention)") for swap in interestRateCurve.swaps.curvepoints { print("Swap tenor = \(swap.tenor) maturitydate = \(swap.maturitydate) and parrate = \(swap.parrate)") } } let xmltxt = "<?xml version=\"1.0\" standalone=\"yes\" ?><interestRateCurve><effectiveasof>2016-07-05</effectiveasof><currency>GBP</currency><baddayconvention>M</baddayconvention><deposits><daycountconvention>ACT/365</daycountconvention><snaptime>2016-07-04T15:00:00.000Z</snaptime><spotdate>2016-07-05</spotdate><calendars><calendar>none</calendar></calendars><curvepoint><tenor>1M</tenor><maturitydate>2016-08-05</maturitydate><parrate>0.004848</parrate></curvepoint><curvepoint><tenor>2M</tenor><maturitydate>2016-09-05</maturitydate><parrate>0.005009</parrate></curvepoint><curvepoint><tenor>3M</tenor><maturitydate>2016-10-05</maturitydate><parrate>0.005163</parrate></curvepoint><curvepoint><tenor>6M</tenor><maturitydate>2017-01-05</maturitydate><parrate>0.006185</parrate></curvepoint><curvepoint><tenor>1Y</tenor><maturitydate>2017-07-05</maturitydate><parrate>0.008459</parrate></curvepoint></deposits><swaps><fixeddaycountconvention>ACT/365</fixeddaycountconvention><floatingdaycountconvention>ACT/365</floatingdaycountconvention><fixedpaymentfrequency>6M</fixedpaymentfrequency><floatingpaymentfrequency>6M</floatingpaymentfrequency><snaptime>2016-07-04T15:00:00.000Z</snaptime><spotdate>2016-07-05</spotdate><calendars><calendar>none</calendar></calendars><curvepoint><tenor>2Y</tenor><maturitydate>2018-07-05</maturitydate><parrate>0.00497</parrate></curvepoint><curvepoint><tenor>3Y</tenor><maturitydate>2019-07-05</maturitydate><parrate>0.00504</parrate></curvepoint><curvepoint><tenor>4Y</tenor><maturitydate>2020-07-05</maturitydate><parrate>0.00539</parrate></curvepoint><curvepoint><tenor>5Y</tenor><maturitydate>2021-07-05</maturitydate><parrate>0.00589</parrate></curvepoint><curvepoint><tenor>6Y</tenor><maturitydate>2022-07-05</maturitydate><parrate>0.00656</parrate></curvepoint><curvepoint><tenor>7Y</tenor><maturitydate>2023-07-05</maturitydate><parrate>0.00732</parrate></curvepoint><curvepoint><tenor>8Y</tenor><maturitydate>2024-07-05</maturitydate><parrate>0.00809</parrate></curvepoint><curvepoint><tenor>9Y</tenor><maturitydate>2025-07-05</maturitydate><parrate>0.00879</parrate></curvepoint><curvepoint><tenor>10Y</tenor><maturitydate>2026-07-05</maturitydate><parrate>0.0094</parrate></curvepoint><curvepoint><tenor>12Y</tenor><maturitydate>2028-07-05</maturitydate><parrate>0.01041</parrate></curvepoint><curvepoint><tenor>15Y</tenor><maturitydate>2031-07-05</maturitydate><parrate>0.01134</parrate></curvepoint><curvepoint><tenor>20Y</tenor><maturitydate>2036-07-05</maturitydate><parrate>0.01186</parrate></curvepoint><curvepoint><tenor>25Y</tenor><maturitydate>2041-07-05</maturitydate><parrate>0.01179</parrate></curvepoint><curvepoint><tenor>30Y</tenor><maturitydate>2046-07-05</maturitydate><parrate>0.01162</parrate></curvepoint></swaps></interestRateCurve>" timetest("Interest Rate Curve XMLParser", block:{printcurve(xmltxt)})

Xcode 8 Beta output




MarKit data are zipped archives. Here is a sample shell script to retrieve and unzip the interest rate curve data. Alternative way is to use other unzip Swift Framework and unzip the url data contents in the code.
getMakit.sh    Select all
#!/bin/bash mkdir -p ~/Downloads/MarKit cd ~/Downloads/MarKit for date in "20160704" "20160705" ;do for ccy in "USD" "GBP" "EUR" "JPY" "CHF" "CAD" "AUD" "NZD" "SGD" "HKD" ;do echo InterestRates_${ccy}_${date}.zip curl -OL https://www.markit.com/news/InterestRates_${ccy}_${date}.zip unzip -o InterestRates_${ccy}_${date}.zip done done


After download and unzip in a folder, it is possible to run a series of XMLParser like this code below

testMarkit2_main.swift    Select all
func test2(url:Foundation.URL) { //func test2(url url:Foundation.NSURL) { // Swift 2 let interestRateCurve = InterestRateCurve() interestRateCurve.startParsing(url:url) print("interestRateCurve currency is \(interestRateCurve.currency) and effectiveasof is \(interestRateCurve.effectiveasof)") print(String(repeating: "-", count: 65)) // SE-0130 Xcode 8 beta 6 // print(String(repeating: ("-" as Character), count: 65)) // Xcode 8 beta 1 print("Deposits spotdate is \(interestRateCurve.deposits.spotdate) daycountconvention is \(interestRateCurve.deposits.daycountconvention)") for depo in interestRateCurve.deposits.curvepoints { print("Depo tenor = \(depo.tenor) maturitydate = \(depo.maturitydate) and parrate = \(depo.parrate)") } print("") print("Swaps spotdate is \(interestRateCurve.swaps.spotdate) fixeddaycountconvention is \(interestRateCurve.swaps.fixeddaycountconvention) floatingdaycountconvention is \(interestRateCurve.swaps.floatingdaycountconvention)") for swap in interestRateCurve.swaps.curvepoints { print("Swap tenor = \(swap.tenor) maturitydate = \(swap.maturitydate) and parrate = \(swap.parrate)") } } for date in ["20160704","20160705"] { for ccy in ["USD","GBP"] { let url = Foundation.URL(fileURLWithPath: NSHomeDirectory() + "/Downloads/MarKit/InterestRates_\(ccy)_\(date).xml") // let url = Foundation.NSURL(fileURLWithPath: NSHomeDirectory() + "/Downloads/MarKit/InterestRates_\(ccy)_\(date).xml") // Swift 2 timetest("InterestRates_\(ccy)_\(date).xml", block:{test2(url:url)}) } }


Below is the playground code (swift 3 Xcode 8 beta 6 and updated to support swift 4 Xcode 9 beta 2) that make use of the compression library (introduced from iOS9 and macOS10.11) to unzip the MarKit Interest Rate Curve url download and then possibly use the InterestRateCurve() XMLParser to print curve points.

unzip_getMarkit.swift    Select all
import Foundation import Compression func timetest(_ note: String, block: () -> Void) //func timetest(note: String, block: () -> Void) // Swift 2 { let date = Date() // let date = NSDate() // Swift 2 block() let timeInterval = Date().timeIntervalSince(date) // let timeInterval = NSDate().timeIntervalSinceDate(date) // Swift 2 print("Test:", note); print("Elapsed time: \(String(format: "%.2f", timeInterval)) s") } func getMarkit(ccy:String, date:String) -> String? { //func getMarkit(ccy ccy:String, date:String) -> String? { // Swift 2 var bytes:[UInt8] = [] var dst:[UInt8] = [] let markitURL = Foundation.URL(string: "https://www.markit.com/news/InterestRates_"+ccy+"_"+date+".zip") // let markitURL = Foundation.NSURL(string: "https://www.markit.com/news/InterestRates_"+ccy+"_"+date+".zip") // Swift 2 guard let markitData = try? Data(contentsOf: markitURL!) , markitData.subdata(in: 0..<4).elementsEqual([0x50,0x4b,0x03,0x04]) // find <504b0304> little-endian <04034b50> // guard let markitData = try? Data(contentsOf: markitURL!) where markitData.subdata(in: 0..<4).elementsEqual([0x50,0x4b,0x03,0x04]) // Xcode 8 beta 1 // guard let markitData = NSData(contentsOfURL: markitURL!) where markitData.subdataWithRange(NSMakeRange(0,4)).isEqualToData(NSData(bytes:[0x50,0x4b,0x03,0x04] as [UInt8], length:4)) // Swift 2 else { return nil } // if let data:NSData = markitData as NSData { // Swift 2 or Xcode 8 beta 1 bytes = Array(UnsafeBufferPointer(start: (markitData as NSData).bytes.bindMemory(to: UInt8.self, capacity: markitData.count), count: markitData.count)) // SE-0107 Xcode 8 beta 6 and convert from markitData:Data to [UInt8] // if convert from data:NSData to [UInt8] // bytes = Array(UnsafeBufferPointer(start: UnsafePointer<UInt8>(data.bytes.assumingMemoryBound(to: UInt8.self)), count: data.length)) // Xcode 8 beta 6 // bytes = Array(UnsafeBufferPointer(start: UnsafePointer<UInt8>(data.bytes), count: data.length)) // Swift 2 or Xcode 8 beta 1 // } // Swift 2 or Xcode 8 beta 1 var firstcentralheader: Int = 0 var offset: Int = 0 var nextoffset: Int = 0 var found:Bool = false let xmlfilenameprefix = "InterestRates" var xmlfilenamelength: Int = 0 var xmlfilename : String = "" for i in 1...bytes.count-4 { // Central directory file header signature = 0x02014b50 if bytes[i..<i+4].elementsEqual([0x50,0x4b,0x01,0x02]) { // find <504b0102> little-endian <02014b50> if firstcentralheader == 0 { firstcentralheader = i } // Central directory : 28 2 File name length (n) let filenamelength = bytes[(i+28)..<(i+28+2)] .enumerated() // .enumerate() // Swift 2 .map { let (index, element) = $0; return Int(Double(element) * pow(256,Double(index))) } // Xcode 9 beta 2 swift 4 and swift 3, SE-0110 which will be back-out in official release // .map { (index, element) in return Int(Double(element) * pow(256,Double(index))) } // Xcode 8 beta 6 // .map { (index, element) in return Int(element) * Int(pow(Double(256),Double(index))) } // Swift 2 .reduce(0, +) // Xcode 8 beta 6 // .reduce(0, combine: +) // Swift 2 or Xcode 8 beta 1 let filename = bytes[(i+46)..<(i+46+filenamelength)].reduce("", { $0 + String(format: "%c", $1)}) // Xcode 8 beta 6 // let filename = bytes[(i+46)..<(i+46+filenamelength)].reduce("", combine: { $0 + String(format: "%c", $1)}) // Swift 2 or Xcode 8 Beta 1 if !found && filename.hasPrefix(xmlfilenameprefix) { // Central directory : 42 4 Relative offset of local file header offset = bytes[(i+42)..<(i+42+4)] .enumerated() // .enumerate() // Swift 2 .map { let (index, element) = $0; return Int(Double(element) * pow(256,Double(index))) } // Xcode 9 beta 2 swift 4 and swift 3, SE-0110 which will be back-out in official release // .map { (index, element) in return Int(Double(element) * pow(256,Double(index))) } // Xcode 8 beta 6 // .map { (index, element) in return Int(element) * Int(pow(Double(256),Double(index))) } // Swift 2 .reduce(0, +) // Xcode 8 beta 6 // .reduce(0, combine: +) // Swift 2 or Xcode 8 beta 1 // Local file header : offset+26 2 File name length (n) xmlfilenamelength = bytes[(offset+26)..<(offset+26+2)] .enumerated() // .enumerate() // Swift 2 .map { let (index, element) = $0; return Int(Double(element) * pow(256,Double(index))) } // Xcode 9 beta 2 swift 4 and swift 3, SE-0110 which will be back-out in official release // .map { (index, element) in return Int(Double(element) * pow(256,Double(index))) } // Xcode 8 beta 6 // .map { (index, element) in return Int(element) * Int(pow(Double(256),Double(index))) } // Swift 2 .reduce(0, +) // Xcode 8 beta 6 // .reduce(0, combine: +) // Swift 2 or Xcode 8 beta 1 xmlfilename = bytes[(i+46)..<(i+46+xmlfilenamelength)].reduce("", { $0 + String(format: "%c", $1)}) // Xcode 8 beta 6 // xmlfilename = bytes[(i+46)..<(i+46+xmlfilenamelength)].reduce("", combine: { $0 + String(format: "%c", $1)}) // Swift 2 or Xcode 8 beta 1 found = true } else { if found && nextoffset == 0 { nextoffset = bytes[(i+42)..<(i+42+4)] .enumerated() // .enumerate() // Swift 2 .map { let (index, element) = $0; return Int(Double(element) * pow(256,Double(index))) } // Xcode 9 beta 2 swift 4 and swift 3, SE-0110 which will be back-out in official release // .map { (index, element) in return Int(Double(element) * pow(256,Double(index))) } // Xcode 8 beta 6 // .map { (index, element) in return Int(element) * Int(pow(Double(256),Double(index))) } // Swift 2 .reduce(0, +) // Xcode 8 beta 6 // .reduce(0, combine: +) // Swift 2 or Xcode 8 beta 1 break } } } } //print(xmlfilename) // Local file header : offset+30 n File name let bytes2 = Array(bytes[(offset+30+xmlfilenamelength)..<(nextoffset > 0 ? nextoffset :firstcentralheader)]) if bytes2.count > 0 { dst = [UInt8](repeating: 0, count: bytes2.count*10) // destination buffer // dst = [UInt8](count: bytes2.count*10, repeatedValue: 0 ) // destination buffer // Swift 2 // use compression library function (iOS9 or macOS 10.11 or above) if #available(OSX 10.11, iOS 9.0, *) { let size = compression_decode_buffer(&dst, dst.count, bytes2, bytes2.count, nil, COMPRESSION_ZLIB) if size > 0 { if let outxml = String(bytes: dst[0..<size], encoding: String.Encoding.ascii) { // if let outxml = String(bytes: dst[0..<size], encoding: NSASCIIStringEncoding) { // Swift 2 return outxml } } } } return nil } timetest("Get MarKit xml", block:{ for date in ["20160808","20160809"] { for ccy in ["USD","GBP"] { if let xml = getMarkit(ccy: ccy, date: date) { print(xml) // printcurve(xml) // if using the InterestRateCurve() XMLParser to print curve points } } } })