Thursday, June 10, 2021

How to download WWDC2021 videos and subtitles

The scripts for previous WWDC are here https://iphonesdkdev.blogspot.com/2020/10/how-to-download-wwdc2020-videos-and.html

https://iphonesdkdev.blogspot.com/2017/07/how-to-fetch-wwdc-2017-video-subtitle.html

Create and Run this script wwdc2021_fetch_srt.sh to fetch WWDC2021 subtitle


wwdc2021_fetch_en.srt.sh    Select all
#!/bin/bash # @Last Modified by: javacom # @Last Modified time: 2021-06-10 WWDC_YEAR=2021; # WWDC_SESSION_PREFIX=https://developer.apple.com/videos/play/wwdc$WWDC_YEAR; WWDC_LOCAL_DIR=$(basename $WWDC_SESSION_PREFIX); detect_video_m3u8 () { local session_url=$WWDC_SESSION_PREFIX/$SESSION_ID/; local session_html=$(curl -s $session_url); local video_url=$(echo "$session_html" | grep .m3u8 | grep $SESSION_ID | head -n1 | sed "s#.*\"\(https://.*m3u8\)\".*#\1#"); echo "$session_html" | grep .mp4 | grep $SESSION_ID | sed "s#.*\"\(https://.*mp4\).*\".*#\1#" | while read mp4_url; do local mp4_filename=$(basename $mp4_url | cut -d. -f1); local srt_filename=$mp4_filename.en.srt; # local srt_filename=$mp4_filename.zh.srt; echo "> Subtitle local: $WWDC_LOCAL_DIR/$srt_filename" >&2; > $WWDC_LOCAL_DIR/$srt_filename; done echo "$video_url"; echo "> Video: $video_url" >&2; } detect_subtitle_m3u8 () { local video_url=$1; # en subtitle local subtitle_uri=$(curl -s $video_url | grep "LANGUAGE=\"eng\"" | grep "TYPE=SUBTITLES" | sed "s#.*URI=\"\(.*\)\"#\1#" | sed "s#.FORCED=NO##"); # zh subtitle #local subtitle_uri=$(curl -s $video_url | grep "LANGUAGE=\"zh\"" | sed "s#.*URI=\"\(.*\)\"#\1#"); local subtitle_url=$subtitle_uri; [[ "$subtitle_uri" != http* ]] && { subtitle_url=$(dirname $video_url)/$subtitle_uri; } echo "$subtitle_url"; echo "> Subtitle: $subtitle_url" >&2; } download_subtitle_contents () { local subtitle_url=$1; echo "> Downloading... " local subtitle_base_url=$(dirname $subtitle_url); curl -s $subtitle_url | grep "webvtt" | while read webvtt; do local subtitle_webvtt=$subtitle_base_url/$webvtt; #echo "- get $subtitle_webvtt"; local subtitle_content=$(curl -s $subtitle_webvtt); # en subtitle ls $WWDC_LOCAL_DIR/"wwdc$WWDC_YEAR"-"$SESSION_ID"*.en.srt | while read srt_file; do # zh subtitle # ls $WWDC_LOCAL_DIR/"wwdc$WWDC_YEAR"-"$SESSION_ID"*.zh.srt | while read srt_file; do echo "$subtitle_content" >> $srt_file; done done } main () { [ ! -d $WWDC_LOCAL_DIR ] && { mkdir $WWDC_LOCAL_DIR; } #Year 2020/2021 change {3\} to {3,5\} curl -s $WWDC_SESSION_PREFIX | grep /videos/play/wwdc$WWDC_YEAR | sed "s#.*/videos/play/wwdc$WWDC_YEAR/\([0-9]\{3,5\}\).*#\1#" | sort | uniq | while read SESSION_ID; do #echo "SESSION_ID is" $SESSION_ID local video_url=$(detect_video_m3u8 $SESSION_ID); local subtitle_url=$(detect_subtitle_m3u8 $video_url); download_subtitle_contents $subtitle_url; done } main;




Run this shell script to format as SRT subtitle

shellscript.sh    Select all
WWDC_YEAR=2021; # cd wwdc$WWDC_YEAR mkdir -p sd mkdir -p hd for i in *_sd.??.srt; do sed -e '/WEBVTT/d;/X-TIMESTAMP/d;s/align.middle line.*$//;' $i | awk '/^[0-9]{2}:[0-9]{2}:/ {seen[$0]++; skipduplicated=0} {if (seen[$0]>1) skipduplicated=1; if (!skipduplicated) print $0}' | awk -v RS="" '{gsub("\n", "-Z"); print}' | awk '$0 !~/^WEB/ {print $0}' | uniq | awk '{printf "\n%s-Z%s", NR,$0 }' | awk -v ORS="\n\n" '{gsub("-Z", "\n"); print}' | sed -e 's/.A:middle$//g;s/&gt;/>/g;s/&lt;/</g;1,2d;' > sd/$i; done for i in *_hd.??.srt; do sed -e '/WEBVTT/d;/X-TIMESTAMP/d;s/align.middle line.*$//;' $i | awk '/^[0-9]{2}:[0-9]{2}:/ {seen[$0]++; skipduplicated=0} {if (seen[$0]>1) skipduplicated=1; if (!skipduplicated) print $0}' | awk -v RS="" '{gsub("\n", "-Z"); print}' | awk '$0 !~/^WEB/ {print $0}' | uniq | awk '{printf "\n%s-Z%s", NR,$0 }' | awk -v ORS="\n\n" '{gsub("-Z", "\n"); print}' | sed -e 's/.A:middle$//g;s/&gt;/>/g;s/&lt;/</g;1,2d;' > hd/$i; done




Run this script wwdc2021_fetch_mp4.sh to download all mp4 (HD and SD) videos (also works for 2021)

wwdc2021_fetch_mp4.sh    Select all
#!/bin/bash # @Last Modified by: javacom # @Last Modified time: 2021-06-10 WWDC_YEAR=2021; # WWDC_SESSION_PREFIX=https://developer.apple.com/videos/play/wwdc$WWDC_YEAR; WWDC_LOCAL_DIR=$(basename $WWDC_SESSION_PREFIX); download_mp4_video () { local session_url=$WWDC_SESSION_PREFIX/$SESSION_ID/; local session_html=$(curl -s $session_url); local video_url=$(echo "$session_html" | grep .m3u8 | grep $SESSION_ID | head -n1 | sed "s#.*\"\(https://.*m3u8\)\".*#\1#"); echo "$session_html" | grep .mp4 | grep $SESSION_ID | sed "s#.*\"\(https://.*mp4\).*\".*#\1#" | while read mp4_url; do local mp4_filename=$(basename $mp4_url); if [ -e $WWDC_LOCAL_DIR/$mp4_filename ] then echo "> MP4 already existed : $WWDC_LOCAL_DIR/$mp4_filename" >&2; echo "> To resume broken download use curl -C - --connect-timeout 1200 -o $WWDC_LOCAL_DIR/$mp4_filename $mp4_url" >&2; echo " " >&2; else echo "> MP4 Downloading... : $mp4_url" >&2; curl --connect-timeout 120 -o $WWDC_LOCAL_DIR/$mp4_filename $mp4_url fi done } main () { [ ! -d $WWDC_LOCAL_DIR ] && { mkdir $WWDC_LOCAL_DIR; } #Year 2020/2021 change {3\} to {3,5\} curl -s $WWDC_SESSION_PREFIX | grep /videos/play/wwdc$WWDC_YEAR | sed "s#.*/videos/play/wwdc$WWDC_YEAR/\([0-9]\{3,5\}\).*#\1#" | sort | uniq | while read SESSION_ID; do download_mp4_video $SESSION_ID; done } main;






Run this script to rename the video or subtitles to proper title (HD & SD) videos

wwdc2021_rename_title.sh    Select all
#!/bin/sh # @Last Modified by: javacom # @Last Modified time: 2021-6-10 # For WWDC2021 video files, VIDEO=sd or VIDEO=hd WWDC_YEAR=2021; VIDEO=sd; curl -s https://developer.apple.com/videos/wwdc${WWDC_YEAR}/ | grep -B1 "video-title" | sed -e "s#[[:space:]]\{10,\}<a href=./videos/play/wwdc202./\([0-9]\{3,5\}\).*#\[ -f \"wwdc${WWDC_YEAR}-\1_${VIDEO}.mp4\" \] \&\& mv \"wwdc${WWDC_YEAR}-\1_${VIDEO}.mp4\" \"wwdc${WWDC_YEAR}-\1_${VIDEO}_#" -e "s/[\@:’\'\,?]//g" -e "s/\"\(The.*\)\"/\1/" -e 's#.*video-title..\(.*\)\(</h4>\)#\1.mp4"#' -e '/--/d' | sed '/^\[ -f/{N;s/\n//;}' | sed "s#[\/@:’\'\,]#_#g" | /bin/bash # For WWDC2021 English subtitle files, VIDEO=sd or VIDEO=hd WWDC_YEAR=2021; VIDEO=sd; curl -s https://developer.apple.com/videos/wwdc${WWDC_YEAR}/ | grep -B1 "video-title" | sed -e "s#[[:space:]]\{10,\}<a href=./videos/play/wwdc202./\([0-9]\{3,5\}\).*#\[ -f \"wwdc${WWDC_YEAR}-\1_${VIDEO}.en.srt\" \] \&\& mv \"wwdc${WWDC_YEAR}-\1_${VIDEO}.en.srt\" \"wwdc${WWDC_YEAR}-\1_${VIDEO}_#" -e "s/[\@:’\'\,?]//g" -e "s/\"\(The.*\)\"/\1/" -e 's#.*video-title..\(.*\)\(</h4>\)#\1.en.srt"#' -e '/--/d' | sed '/^\[ -f/{N;s/\n//;}' | sed "s#[\/@:’\'\,]#_#g" | /bin/bash






# Example nohup commnad for Linux Download

nohup bash -c 'cd $HOME/Downloads/WWDC/; ./wwdc2021_fetch_mp4.sh' &> nohup.wwdc2021.mp4.out &

nohup bash -c 'cd $HOME/Downloads/WWDC/; ./wwdc2021_fetch_en.srt.sh' &> nohup.wwdc2021.en.srt.out &

# or use screen utility to download
screen -S wwdcdownloadmp4
bash wwdc2021_fetch_mp4.sh



Thursday, April 8, 2021

How to write subrountine call in macOS assembly code for x86_64 and arm64

(1) Demo the subrountine call in macOS assembly code for x86_64 and arm64. For M1 Mac with Rosetta 2 installed, it can compile and run x86_64 and arm64 binary after installation of Xcode
callfactorial.c  Select all
/* * An application that illustrates calling the factorial function defined elsewhere. */ #include <stdio.h> #include <inttypes.h> uint64_t factorial(unsigned n); int main() { for (unsigned i = 0; i < 20; i++) { printf("factorial(%2u) = %llu\n", i, factorial(i)); } }


factorial.s  Select all
# ---------------------------------------------------------------------- # A 64-bit recursive implementation of the function # # uint64_t factorial(unsigned n) # # implemented recursively with x86_64 and arm64 assembly code # ----------------------------------------------------------------------- .globl _factorial .text #ifdef __arm64__ .align 4 #endif _factorial: #ifdef __x86_64__ cmp $1, %rdi # n <= 1? jnbe L1 # if not, go do a recursive call mov $1, %rax # otherwise return 1 ret #endif #ifdef __arm64__ cmp x8, #1 //# n > 1? b.gt L1 //# if yes, go do a recursive call mov x0, #1 //# otherwise return 1 ret #endif L1: #ifdef __x86_64__ push %rdi # save n on stack (also aligns %rsp!) dec %rdi # n-1 call _factorial # factorial(n-1), result goes in %rax pop %rdi # restore n imul %rdi, %rax # n * factorial(n-1), stored in %rax ret #endif #ifdef __arm64__ STP X8, LR, [SP, #-16]! //# push x8 and LR(x30) // LR is to return from subroutine subs x8, x8, #1 //# n-1 bl _factorial //# factorial(n-1), result goes in x0 LDP X8, LR, [SP], #16 //# pop x8 and LR(x30) mul x0, x0, x8 //# n * factorial(n-1), stored in x0 ret #endif


(2) To compile with -g and codesign the program so as to debug in lldb under macOS.
shell script  Select all
# To compile and codesign x86_64 version clang factorial.s callfactorial.c -g -o callfactorial_arm64 -arch x86_64 && codesign --entitlement entitlements --force -s - callfactorial_x86_64 # To compile and codesign arm64 version clang factorial.s callfactorial.c -g -o callfactorial_arm64 -arch arm64 && codesign --entitlement entitlements --force -s - callfactorial_arm64


(3) To debug using lldb
shell script  Select all
lldb callfactorial_x86_64 # or lldb callfactorial_arm64 # lldb debug session for arm64 - useful commands (lldb) breakpoint set --name main --name factorial (lldb) breakpoint list (lldb) run (lldb) step (lldb) po i (lldb) reg read x0 x8 lr pc (lldb) reg read -f t cpsr # lldb debug session for x86_64 - useful commands (lldb) reg read -f d rax rdi rflags (lldb) reg read -f t rflags # print the address value in the stackpointer for x86_64 (lldb) p *(int **)$sp # hint: to search lldb command history use ctrl-r


Saturday, April 3, 2021

How to use inline assembly language for M1 Mac for x86_64 and arm64

(1) Demo the inline assembly code for x86_64 and arm64
add-inline.c  Select all
#include <stdio.h> // compile with command line // clang add-inline.c -o add-inline_x86_64 -arch x86_64 // clang add-inline.c -o add-inline_arm64 -arch arm64 // disassemble with // otool -otV add-inline_x86_64 // otool -otV add-inline_arm64 int main(int argc, const char * argv[]) { int a = 10; int b = 25; int ans = 0; #ifdef __x86_64__ __asm__( "add %2,%1;\n" // %1 += %2 add source to destination "mov %1,%0;\n" // move data from %1 to %0 : "=r"(ans) : "r"(a), "r"(b) : ); #endif #ifdef __arm64__ __asm__( "add %w0,%w1,%w2;\n" // load %w0 = %w1 + %w2 : "=r"(ans) : "r"(a), "r"(b) : ); #endif printf("The answer is %d\n",ans); return 0; }


(2) Demo the assembly source code as a separate function for x86_64 and arm64
add-main.c  Select all
#include <stdio.h> // compile with command line // clang add-main.c add.s -o add-main_x86_64 -arch x86_64 // clang add-main.c add.s -o add-main_arm64 -arch arm64 #include <stdio.h> int add(int x, int y); int main(int argc, const char * argv[]) { int ans = add(15,40); printf("The answer is %d\n",ans); return 0; }


add.s  Select all
.text .globl _add .align 2 _add: #ifdef __x86_64__ add %esi,%edi // %edi += %esi, source is the first mov %edi,%eax // move data from %edi to %eax // x86_64 calling convention // rdi, rsi, rdx, rcx, r8, r9 // The 32-bit general purpose registers are edi, esi, edx, ecx, r8d, r9d instead. // The 16-bit general purpose registers are di, si, dx, cx, r8w, r9w instead. // The syscall number is placed in rax // see https://sigsegv.pl/osx-bsd-syscalls/ // Return value is in rax #endif #ifdef __arm64__ add w0,w0,w1 // load w0 with w0+w1, destination is the first // calling convention for arm64 // x0,x1,x2,x3,x4,x5,x6,x7 or r0 to r7 // The 32-bit general purpose registers are w0-w7 instead. // The syscall number is placed in r8 // Return value is in x0 // see https://wiki.cdot.senecacollege.ca/wiki/Syscalls #endif ret




entitlements  Select all
<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> <key>com.apple.security.get-task-allow</key> <true/> </dict> </plist>


(3) Compile with
clang add-inline.c -o add-inline_x86_64 -arch x86_64
clang add-inline.c -o add-inline_arm64 -arch arm64
clang add-main.c add.s -o add-main_x86_64 -arch x86_64
clang add-main.c add.s -o add-main_arm64 -arch arm64


(4) Disassemble with (e.g.)
otool -otV add-inline_arm64

(5) Install Rosetta 2 on M1 to run the x86_64 version (e.g.)
./add-main_x86_64

(6) codesign and debug with lldb (e.g.)
clang -g -o add-inline_x86_64 add-inline.c -arch x86_64
codesign --entitlement entitlements --force -s - add-inline_x86_64
lldb add-inline_x86_64
(lldb) breakpoint set --file add-inline.c --line 7


(7) floating point example for x86_64 and arm64
compile with
clang sum.s callsum.c -o callsum_x86_64 -arch x86_64
clang sum.s callsum.c -o callsum_arm64 -arch arm64
Debug and codesign similar to above example
callsum.c  Select all
/* * callsum.c * * Illustrates how to call the sum.s function wrote in assembly language. */ // clang sum.s callsum.c -o callsum_x86_64 -arch x86_64 // clang sum.s callsum.c -o callsum_arm64 -arch arm64 #include <stdio.h> double sum(double[], unsigned); int main() { double test[] = { 40.5, 26.7, 21.9, 1.5, -40.5, -23.4 }; printf("%20.7f\n", sum(test, 6)); printf("%20.7f\n", sum(test, 2)); printf("%20.7f\n", sum(test, 0)); printf("%20.7f\n", sum(test, 3)); return 0; }


sum.s  Select all
# ----------------------------------------------------------------------- # A 64-bit function that returns the sum of the elements in a # floating-point array for x86_64 and arm64. The function has prototype: # # double sum(double[] array, unsigned length) # ----------------------------------------------------------------------- .global _sum .text .align 2 _sum: #ifdef __x86_64__ xorpd %xmm0, %xmm0 // initialize the sum to 0 // floats are passed in xmm0 cmp $0, %rsi // special case for length = 0 je done #endif #ifdef __arm64__ movi d0, #0 // initialize the sum to 0 // floats are passed in s0-7 and doubles in the d0-7 registers. cmp x1, #0 // special case for length = 0 b.eq done #endif next: #ifdef __x86_64__ addsd (%rdi), %xmm0 // add in the current array element, return floating point value in xmm0 add $8, %rdi // move to next array element dec %rsi // count down jnz next // if not done counting, continue #endif #ifdef __arm64__ ldr d16, [x0] // load the float into d16 // floats in s0-7 and doubles in the d0-7 registers. fadd d0, d0, d16 // add in the current array element, return floating point value in d0 add x0, x0, #8 // move to next array element subs x1, x1, #1 // count down cbnz w1, next // if not done counting, continue #endif done: ret


Wednesday, December 23, 2020

How to install Parallel Desktop and Windows 10 on Mac M1

Mac M1 can install Parallel Desktop Preview and run x64 Windows app

1) Install PD for M1 Mac from https://www.parallels.com/blogs/parallels-desktop-apple-silicon-mac/ and click Try Technical Preview. Need to login and register.

Direct download link
MD5 checksum 5fc71f89fcc67ed13bac1995604a962f
Activation key You have to register and login to activate the product

2) Install Windows 10 Client ARM64 Insider Preview download from https://www.microsoft.com/en-us/software-download/windowsinsiderpreviewARM64. Need to login and register as Insider Preview Program.
Direct download link>
>
3) Update Win10 ARM to version 21277 which can install and run x64 Windows programs.
Have to login Windows Insider Program and set to "Dev Channel" to recieve preview updates.


Sunday, December 20, 2020

Buid docker image for M1 Mac with development tools and vnc

(1) With the release of docker preview for Mac M1, it is now possible to build some arm64 images.
Docker M1 preview
https://desktop.docker.com/mac/m1preview/Docker-AppleSilicon-Preview7.dmg

(2) Install the above preview and then use terminal the tools. The build script is based on Android SDK docker image for x64 and modify it for arm64. Moreover, swift 5.3.1, clang 10.0.0, kotlin 1.4.21, gradle 6.7.1, gcc 9.3.0, java 11, firefox, visual studio code and codeblocks are added for development purpose.

(3) Shell script run in Mac docker info # check if docker is installed properly in Mac M1
mkdir -p ${HOME}/android_sdk_vnc
cd ${HOME}/android_sdk_vnc
curl -O https://raw.githubusercontent.com/thyrlian/AndroidSDK/master/android-sdk/sshd-banner
curl -O https://raw.githubusercontent.com/thyrlian/AndroidSDK/master/android-sdk/supervisord.conf
curl -O https://raw.githubusercontent.com/thyrlian/AndroidSDK/master/android-sdk/vnc/supervisord_vncserver.conf
curl -O https://raw.githubusercontent.com/thyrlian/AndroidSDK/master/android-sdk/vnc/vncpass.sh
curl -O https://raw.githubusercontent.com/thyrlian/AndroidSDK/master/android-sdk/vnc/watchdog.sh

(4) Create the following Dockfile
Dockerfile    Select all
cat >Dockerfile <<'HEREEOF' # ====================================================================== # # Android SDK Docker Image # ====================================================================== # # Base image # ---------------------------------------------------------------------- # FROM arm64v8/ubuntu:20.04 # install essential tools RUN apt-get update && \ apt-get dist-upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends git wget wget2 zip unzip && \ apt-get install -y build-essential vim # install Android SDK RUN DEBIAN_FRONTEND=noninteractive apt-get install -y android-sdk && \ rm -f /usr/lib/android-sdk/build-tools/27.0.1 # install Android SDK cmdline tools ARG ANDROID_SDK_VERSION=6858069 ENV ANDROID_SDK_ROOT /usr/lib/android-sdk RUN wget2 -q https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_VERSION}_latest.zip && \ unzip commandlinetools*linux*${ANDROID_SDK_VERSION}*.zip && \ mv cmdline-tools tools && \ mkdir -p ${ANDROID_SDK_ROOT}/cmdline-tools && \ mv tools ${ANDROID_SDK_ROOT}/cmdline-tools/ && \ (cd /usr/lib/android-sdk/cmdline-tools; ln -s tools latest) && \ rm -f commandlinetools*linux*.zip # download and install Gradle ARG GRADLE_VERSION=6.7.1 ARG GRADLE_DIST=bin RUN wget2 -q https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-${GRADLE_DIST}.zip && \ rm -fr /usr/share/gradle && \ unzip gradle*.zip -d /usr/share && \ rm gradle*.zip && \ mv /usr/share/gradle-${GRADLE_VERSION} /usr/share/gradle # download and install Kotlin compiler ARG KOTLIN_VERSION=1.4.10 RUN wget2 -q https://github.com/JetBrains/kotlin/releases/download/v${KOTLIN_VERSION}/kotlin-compiler-${KOTLIN_VERSION}.zip && \ unzip *kotlin*.zip && \ mv kotlinc /usr/share && \ rm *kotlin*.zip # install swift5 compiler RUN wget -qO- https://packagecloud.io/install/repositories/swift-arm/release/script.deb.sh | bash RUN apt install -y swiftlang # setup adb server EXPOSE 5037 # set the environment variables ARG JDK_VERSION=1.11.0 ENV JAVA_HOME /usr/lib/jvm/java-${JDK_VERSION}-openjdk-arm64 ENV GRADLE_HOME /usr/share/gradle ENV KOTLIN_HOME /usr/share/kotlinc ENV ANDROID_SDK_ROOT /usr/lib/android-sdk ENV ANDROID_HOME /usr/lib/android-sdk ENV ANDROID_TOOLS /usr/lib/android-sdk/tools ENV ANDROID_PLATFORMS /usr/lib/android-sdk/platforms ENV ANDROID_PLATFORM_TOOLS /usr/lib/android-sdk/platform-tools ENV ANDROID_BUILD_TOOLS /usr/lib/android-sdk/build-tools ENV ANDROID_CMDLINE_TOOLS /usr/lib/android-sdk/cmdline-tools/tools ENV PATH ${PATH}:${GRADLE_HOME}/bin:${KOTLIN_HOME}/bin:${ANDROID_BUILD_TOOLS}/debian:${ANDROID_TOOLS}/bin:${ANDROID_PLATFORM_TOOLS}:${ANDROID_CMDLINE_TOOLS}/bin:${ANDROID_SDK_ROOT}/emulator # install system image and accept licence, installation of emulator will fail for arm64 system RUN yes Y | ${ANDROID_CMDLINE_TOOLS}/bin/sdkmanager --install "platform-tools" "system-images;android-24;google_apis;arm64-v8a" "platforms;android-24" "build-tools;30.0.3" && \ wget https://github.com/qhuyduong/arm_adb/releases/download/v1.0.39-aarch64/adb && \ chmod +x adb && mv adb /usr/lib/android-sdk/platform-tools/ # install and configure SSH server EXPOSE 22 ADD sshd-banner /etc/ssh/ # ADD authorized_keys /tmp/ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server supervisor locales && \ mkdir -p /var/run/sshd /var/log/supervisord && \ locale-gen en en_US en_US.UTF-8 && \ apt-get remove -y locales && apt-get autoremove -y && \ FILE_SSHD_CONFIG="/etc/ssh/sshd_config" && \ echo "\nBanner /etc/ssh/sshd-banner" >> $FILE_SSHD_CONFIG && \ echo "\nPermitUserEnvironment=yes" >> $FILE_SSHD_CONFIG && \ ssh-keygen -q -N "" -f /root/.ssh/id_rsa && \ FILE_SSH_ENV="/root/.ssh/environment" && \ touch $FILE_SSH_ENV && chmod 600 $FILE_SSH_ENV && \ printenv | grep "JAVA_HOME\|GRADLE_HOME\|KOTLIN_HOME\|ANDROID_SDK_ROOT\|LD_LIBRARY_PATH\|PATH" >> $FILE_SSH_ENV && \ echo "\nauth required pam_env.so envfile=$FILE_SSH_ENV" >> /etc/pam.d/sshd && \ FILE_AUTH_KEYS="/root/.ssh/authorized_keys" && \ touch $FILE_AUTH_KEYS && chmod 600 $FILE_AUTH_KEYS && \ for file in /tmp/*.pub; \ do if [ -f "$file" ]; then echo "\n" >> $FILE_AUTH_KEYS && cat $file >> $FILE_AUTH_KEYS && echo "\n" >> $FILE_AUTH_KEYS; fi; \ done && \ (rm /tmp/*.pub 2> /dev/null || true) # install and configure VNC server ENV USER root ENV DISPLAY :1 EXPOSE 5901 ADD vncpass.sh /tmp/ ADD watchdog.sh /usr/local/bin/ ADD supervisord_vncserver.conf /etc/supervisor/conf.d/ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends xfce4 xfce4-goodies xfonts-base dbus-x11 tightvncserver expect && \ apt-get install -y --no-install-recommends firefox codeblocks && \ chmod +x /tmp/vncpass.sh; sync && \ /tmp/vncpass.sh && \ rm /tmp/vncpass.sh && \ apt-get remove -y expect && apt-get autoremove -y && \ FILE_SSH_ENV="/root/.ssh/environment" && \ echo "DISPLAY=:1" >> $FILE_SSH_ENV # install and patch Visual Studio Code RUN apt update && apt install -y libsecret-1-0 libxss1 libgbm1 && apt-get clean RUN wget -O code_arm64.deb https://aka.ms/linux-arm64-deb && dpkg -i code_arm64.deb && rm code_arm64.deb RUN sed 's/BIG-REQUESTS/_IG-REQUESTS/' /usr/lib/aarch64-linux-gnu/libxcb.so.1.1.0 | tee /usr/share/code/libxcb.so.1 > /dev/null ADD supervisord.conf /etc/supervisor/conf.d/ CMD ["/usr/bin/supervisord"] HEREEOF


(5) build the Dockfile
docker build -f Dockerfile -t arm64v8/android-sdk-vnc .


(6) Copy Mac host id_rsa.pub to authorized_keys and start the container
cp ~/.ssh/authorized_keys .
cat ~/.ssh/id_rsa.pub >> authorized_keys


# To start the container as daemon
docker run -d --name android-sdk-vnc -p 5901:5901 -p 2222:22 -p 5037:5037 \
-v $(pwd)/authorized_keys:/root/.ssh/authorized_keys arm64v8/android-sdk-vnc

# To connect the container via ssh
ssh root@<IP address of your Mac M1> -p 2222

# To stop the container
docker container stop android-sdk-vnc


(7) To connect to the vnc of the docker image after starting the container
Use the Mac Finder "Go" -> "Connect To Server" and enter "vnc://0.0.0.0:5901"
For other machines in the network enter "vnc://<IP address of your Mac M1>:5901"

The passwords are in the vncpass.sh when building which are
password="android"
password_view_only="docker"

(8) There is a common problem if you are connecting remotely using Mac Terminal, so the solution is to not forward your locale. Edit /etc/ssh/ssh_config and comment out SendEnv LANG LC_* line.

(9) Testing swift
# attach to the running container
docker exec -it android-sdk-vnc bash
# test
mkdir -p ${HOME}/Projects
cd ${HOME}/Projects
git clone https://github.com/apple/example-package-dealer.git
cd example-package-dealer
swift run Dealer




If you need port forwarding on macOS, see this article https://www.papercut.com/kb/Main/MacPortForwarding

Saturday, December 19, 2020

How to setup debian vm with VNC on New Mac Mini M1

(1) Download the ubuntu kernel and images
cd ${HOME}
mkdir debianvm
curl -o - https://cloud-images.ubuntu.com/releases/focal/release/unpacked/ubuntu-20.04-server-cloudimg-arm64-vmlinuz-generic | gunzip > debianvm/vmlinuz
curl -o debianvm/initrd https://cloud-images.ubuntu.com/releases/focal/release/unpacked/ubuntu-20.04-server-cloudimg-arm64-initrd-generic
curl -o - https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-arm64.tar.gz | tar zxvC debianvm

(2) download the raspbian arm64 image
curl -OL https://downloads.raspberrypi.org/raspios_arm64/images/raspios_arm64-2020-08-24/2020-08-20-raspios-buster-arm64.zip
unzip 2020-08-20-raspios-buster-arm64.zip -d debianvm

(3) Copy and expand the disk to say 38G (Don't use this method to expand disk on external HD)
cp debianvm/focal-server-cloudimg-arm64.img debianvm/raspberry-arm64.img
dd if=/dev/zero of=debianvm/raspberry-arm64.img seek=40000000 obs=1024 count=0

(4) Compile and use the latest vftool which is enhanced to mount multiple disks.
cd ${HOME}
git clone https://github.com/evansm7/vftool
cd vftool && xcodebuild

(5) start the vm say
screen -S vm
cd ${HOME}/debianvm
${HOME}/vftool/build/Release/vftool -k vmlinuz -i initrd -d focal-server-cloudimg-arm64.img -d raspberry-arm64.img -d 2020-08-20-raspios-buster-arm64.img -m 2048 -a "console=hvc0"
Detach the screen using Ctrl-A D

(6) start the tty say
screen -S tty /dev/ttys001

(7) prepare the ubuntu cloud image
mkdir /mnt
mount /dev/vda /mnt
chroot /mnt

touch /etc/cloud/cloud-init.disabled

echo 'root:root' | chpasswd

ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N '' -t rsa
ssh-keygen -f /etc/ssh/ssh_host_dsa_key -N '' -t dsa
ssh-keygen -f /etc/ssh/ssh_host_ecdsa_key -N '' -t ecdsa
ssh-keygen -f /etc/ssh/ssh_host_ed25519_key -N '' -t ed25519

# add fixed IP
cat <<EOF > /etc/netplan/01-dhcp.yaml 
network:
    renderer: networkd
    ethernets:
        enp0s1:
            dhcp4: true
            addresses: [192.168.64.33/24]
    version: 2
EOF

# add IP when login prompt
cat <<EOF >> /etc/issue
enp0s1: \4

EOF

exit
umount /dev/vda



(8) copy the rootfs to raspberry-arm64.img
#Detach the screen using Ctrl-A D
screen -r vm # resume vm screen # stop the vm by ctrl-C and restart the ubuntu vm
cd ${HOME}/debianvm
${HOME}/vftool/build/Release/vftool -k vmlinuz -i initrd -d focal-server-cloudimg-arm64.img -d raspberry-arm64.img -d 2020-08-20-raspios-buster-arm64.img -m 2048 -a "console=hvc0 root=/dev/vda"

#Detach the screen using Ctrl-A D
#Notice the tty number when starting the vm and start the tty say
screen -S tty /dev/ttys001

# login with id root and password root
# and copy the rootfs to the expanded disk raspberry-arm64.img
dd if=/dev/vdc2 of=/dev/vdb bs=64K conv=noerror,sync
exit

(9) prepare the raspberry-arm64 image
# stop the vm and restart the vm
# Control-A D to detach the tty screen # resume vm screen screen -r vm # Control-C to stop the vm cd ${HOME}/debianvm
${HOME}/vftool/build/Release/vftool -k vmlinuz -i initrd -d raspberry-arm64.img -m 2048 -a "console=hvc0"
mkdir /mnt
mount /dev/vda /mnt
chroot /mnt

echo 'root:root' | chpasswd

#!!! delete or comment out the lines in /etc/fstab, or there will be "A start job is running on /dev/disk/.." blocking!!
cat <<EOF > /etc/fstab
proc            /proc           proc    defaults          0       0
#PARTUUID=ad09722e-01  /boot           vfat    defaults          0       2
#PARTUUID=ad09722e-02  /               ext4    defaults,noatime  0       1
EOF

ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N '' -t rsa
ssh-keygen -f /etc/ssh/ssh_host_dsa_key -N '' -t dsa
ssh-keygen -f /etc/ssh/ssh_host_ecdsa_key -N '' -t ecdsa
ssh-keygen -f /etc/ssh/ssh_host_ed25519_key -N '' -t ed25519

mkdir -p /etc/network/interfaces.d/
cat <<EOF > /etc/network/interfaces.d/enp0s1 
allow-hotplug enp0s1
iface enp0s1 inet dhcp
auto lo
iface lo inet loopback
EOF

cat <<EOF >> /etc/dhcpcd.conf
# add fixed IP
interface enp0s1
static ip_address=192.168.64.10/24
EOF

# put mac host ${HOME}/.ssh/id.rsa.pub key here
mkdir -p /root/.ssh
cat <<EOF > /root/.ssh/authorized_keys
ssh-rsa <mac id.rsa.pub key here>
EOF

# add IP when login prompt
cat <<EOF >> /etc/issue
enp0s1: \4

EOF

exit
umount /dev/vda

(10) login raspberry OS arm64 with id root and password root
# stop the vm and restart the vm with
cd ${HOME}/debianvm
${HOME}/vftool/build/Release/vftool -k vmlinuz -i initrd -d raspberry-arm64.img -m 2048 -a "console=hvc0 root=/dev/vda rw"
# set date say
date -s "29 DEC 2020 14:54:00"
resize2fs /dev/vda
dpkg-reconfigure tzdata # set timezone
apt update
systemctl disable rpi-eeprom-update.service
systemctl disable rng-tools.service

(11) install utilities in raspberry os that can handle qcow2 disk image conversion and resize
apt install qemu-utils libguestfs-tools
hostname -I # get ip address

(13) Install Realvncserver and connect from Mac
passwd pi # set user pi password
# login as user pi
login pi
# Use raspi-config to enable SSH and VNC
sudo apt install realvnc-vnc-server
sudo raspi-config
# Choose 5 Interfacing Options -> P2 SSH
# Choose 5 Interfacing Options -> P3 VNC

# enter vnc key, can put this in /etc/rc.local without sudo
sudo vnclicense -add VKUPN-MTHHC-UDHGS-UWD76-6N36A

# setup vncpasswd for user mode, only user mode can run browser
vncpasswd -user

# edit config file to have these two lines for vnc password auth
echo -e "Authentication=VncAuth" | tee -a ~/.vnc/config.d/vncserver-x11
echo -e "Encryption=PreferOff" | tee -a ~/.vnc/config.d/vncserver-x11
cp ~/.vnc/config.d/vncserver-x11 ~/.vnc/config.d/Xvnc
# start service
vncserver

#put these in rc.local to automate after reboot, Raspberry Pi hardware has bundled vnclicense and no need to add it
/usr/bin/vnclicense -add VKUPN-MTHHC-UDHGS-UWD76-6N36A
sudo -H -u pi /usr/bin/vncserver

#check which port is listening
netstat -tlpn | grep -i listen

This would configure VNC in Virtual Mode
# Screen Sharing Connect from Mac
# In Finder, Go -> Connect to Server, enter IP address and port number of the vm e.g.
# Enter the vncpassword to connect, and connection needs port number
vnc://192.168.64.10:5901

For configuring VNC in Service Mode, please refer to https://www.raspberrypi.org/documentation/remote-access/vnc/
For differences in these modes, please refer to https://help.realvnc.com/hc/en-us/articles/360002253238-Understanding-VNC-Server-Modes
#Start or stop the service with:
# systemctl (start|stop) vncserver-x11-serviced.service
#Mark or unmark the service to be started at boot time with:
# systemctl (enable|disable) vncserver-x11-serviced.service
#sudo systemctl start vncserver-x11-serviced.service
For Service Mode the connection from Mac is (no port number), vnc://192.168.64.10

If you need port forwarding on macOS, so that you can vnc connect from other machines on the LAN to this vm, see this article https://www.papercut.com/kb/Main/MacPortForwarding That is run
echo "
rdr pass on lo0 inet proto tcp from any to self port 5910 -> 192.168.64.10 port 5901
rdr pass on en0 inet proto tcp from any to any port 5910 -> 192.168.64.10 port 5901
rdr pass on wlan0 inet proto tcp from any to any port 5910 -> 192.168.64.10 port 5901
" | sudo pfctl -ef -
Then, test connect from other machines to :5901 on the LAN.

# To install Visual Studio Code
wget -O code_arm64.deb https://aka.ms/linux-arm64-deb && sudo dpkg -i code_arm64.deb && rm code_arm64.deb
sed 's/BIG-REQUESTS/_IG-REQUESTS/' /usr/lib/aarch64-linux-gnu/libxcb.so.1.1.0 | sudo tee /usr/share/code/libxcb.so.1 > /dev/null

Demo - VNC to rapberrypi desktop vm screen capture

P.S. Only vm can run realvncserver, docker environment can only run lightweight vncserver e.g. tightvncserver

(14) For qcow2 image, the conversion to raw image is
wget https://cloud.debian.org/images/cloud/buster/20201214-484/debian-10-generic-arm64-20201214-484.qcow2
qemu-img dd -f qcow2 -O raw bs=64k if=debian-10-generic-arm64-20201214-484.qcow2 of=debian-10.img
# scp debian-10.img to the Mac Mini host and prepare a new expanded debian-arm64.img similar to the raspberry-arm64.img as above
# After restarting vm and mounting images, the dd command to extract rootfs is
# Assume debian-arm64.img is vdb and debian-10.img is vdc
dd if=/dev/vdc1 of=/dev/vdb bs=64K conv=noerror,sync
# For debian image, add this to /etc/hosts 127.0.1.1 debian

Thursday, December 3, 2020

How to run docker on the new Mac Mini M1

(1) Reference to this article https://finestructure.co/blog/2020/11/27/running-docker-on-apple-silicon-m1

(2) The problem of this method of running Ubuntu iso live image is not persistent.

(3) So here is the improved method to add persistent storage to this vm.

(4) # First create an empty image data say 50G in Mac Mini
dd if=/dev/zero of=data.img bs=1m count=51200

(5) # Change the command to invoke the vm to include persistent and increase cpu to 2
./vftool -k vmlinuz -i initrd -c focal-desktop-arm64.iso -d data.img -p 2 -m 4096 -a "console=hvc0 persistent rw"
./vftool -k vmlinuz -i initrd -d data.img -c focal-desktop-arm64.iso -p 2 -m 4096 -a "console=hvc0 persistent rw"

P.S.
If you compile the latest vftool, which has added the ability to add more image files and the order of the mounted device will follow the parameters you passed in. The article was written using the first version of the vftool and will mount the data.img as the /dev/vda and the cdrom image as /dev/vdb. So if you use the latest version vftool compiled, you have to check the device name of your mounted data.img.

(6) # Attach the vm and create 3 partitions for the created image
sudo fdisk /dev/vda
# Press (n)(p)(1) to create new partition 1
# Press (enter) to accept starting sector and set size to say +25G
# Press (n)(p)(2) to create new partition 2
# Press (enter) to accept starting sector and set size to say +15G
# Press (n)(p)(3) to create new partition 3
# Accept the default options and starting block and ending block
# Press (w) to write to partition

(6) # Format partition as ext4 with labels casper-rw and home-rw in vm, this create the persistent medium for the rw partition.
sudo mkfs.ext4 /dev/vda1 -L docker-data
sudo mkfs.ext4 /dev/vda2 -L casper-rw
sudo mkfs.ext4 /dev/vda3 -L home-rw

(7) # exit shell and exit vm by ctrl-c
exit

(8) # start vm again and use a new tab using screen command to attach to the vm e.g.
screen -S docker /dev/ttys003

(9) # Create folder to mount /dev/vda1
sudo mkdir -p /mnt/docker-data
# Add this mount commands in /etc/rc.local in order to persist as editing /etc/fstab is ignored in this iso image
cat << EOF | sudo tee -a /etc/rc.local 
#!/bin/sh -e
#
# rc.local
#
# This script is executed at the end of each multiuser runlevel.
# Make sure that the script will "exit 0" on success or any other
# value on error.
#
# In order to enable or disable this script just change the execution
# bits.
#
# By default this script does nothing.

mount /dev/vda1 -t ext4 /mnt/docker-data
exit 0
EOF

cat << EOF | sudo tee -a /etc/systemd/system/rc-local.service
[Unit]
 Description=/etc/rc.local Compatibility
 ConditionPathExists=/etc/rc.local

[Service]
 Type=forking
 ExecStart=/etc/rc.local start
 TimeoutSec=0
 StandardOutput=tty
 RemainAfterExit=yes
 SysVStartPriority=99

[Install]
 WantedBy=multi-user.target
EOF

# enable rc.local see this.
sudo chmod +x /etc/rc.local
sudo systemctl start rc-local.service
sudo systemctl status rc-local.service

(10) Follow the reference article to install docker related packages.
If you have problem when installing docker. Try this = previous verion
apt install docker-ce=5:19.03.14~3-0~ubuntu-focal

(11) # Move docker data-root to the newly created partition
sudo service docker stop
cat << EOF | sudo tee -a /etc/docker/daemon.json 
{
"storage-driver": "overlay2",
"data-root": "/mnt/docker-data"
}
EOF
sudo rsync -aP /var/lib/docker/ /mnt/docker-data
sudo mv /var/lib/docker /var/lib/docker.old
sudo service docker start

(12) Additionally, install docker-compose
sudo add-apt-repository universe
sudo apt install -y python3-pip
sudo pip3 -v install docker-compose

(13) Test docker
docker run hello-world

docker run -e MYSQL_ROOT_PASSWORD=rootpassword -e MYSQL_USER=wpuser -e MYSQL_PASSWORD=wpuserpassword -e MYSQL_DATABASE=wordpressdb --name wordpressdb -d arm64v8/mariadb
docker run -e WORDPRESS_DB_USER=wpuser -e WORDPRESS_DB_PASSWORD=wpuserpassword -e WORDPRESS_DB_NAME=wordpressdb -p 8080:80 --link wordpressdb:mysql --name wordpress -d arm64v8/wordpress
# show IP address
ip addr | grep inet
# test using browser e.g. http://192.168.64.19:8080/

(14) Test docker-compose 1 (wordpress and mysql)
# create docker-compose.yaml
cd $HOME
mkdir my-wordpress
cd my-wordpress
cat > docker-compose.yaml <<EOF
version: '3'
services:
  db:
    image: arm64v8/mariadb
    volumes:
      - "./.data/db:/var/lib/mysql"
    restart: always
    environment:
      MYSQL_ROOT_PASSWORD: rootpassword
      MYSQL_DATABASE: wordpressdb
      MYSQL_USER: wpuser
      MYSQL_PASSWORD: wpuserpassword

  wordpress:
    depends_on:
      - db
    image: arm64v8/wordpress
    links:
      - db:mysql
    ports:
      - "8000:80"
    restart: always
    environment:
      WORDPRESS_DB_HOST: db:3306
      WORDPRESS_DB_NAME: wordpressdb
      WORDPRESS_DB_USER: wpuser
      WORDPRESS_DB_PASSWORD: wpuserpassword
EOF
# Start docker-compose
docker-compose up -d
# test using browser e.g. http://192.168.64.19:8000/
# after testing bring it down
docker-compose down -v

(15) Test docker-compose 2 (mongo-express)
# create docker-compose.yaml
cd $HOME
mkdir my-mongoexpress
cd my-mongoexpress
cat > docker-compose.yaml <<EOF
version: '3.1'
services:
  mongo:
    image: arm64v8/mongo
    volumes:
      - ./data:/data/db
    ports:
      - '27018:27017'
    restart: always
    environment:
      MONGO_INITDB_ROOT_USERNAME: 'root'
      MONGO_INITDB_ROOT_PASSWORD: 'example'
      MONGO_INITDB_DATABASE: 'db'

  mongo-express:
    image: arm64v8/mongo-express
    links:
      - mongo
    ports:
      - "8081:8081"
    restart: always
    environment:
      ME_CONFIG_MONGODB_ADMINUSERNAME: 'root'
      ME_CONFIG_MONGODB_ADMINPASSWORD: 'example'
      ME_CONFIG_MONGODB_ENABLE_ADMIN: 'true'
      ME_CONFIG_MONGODB_AUTH_DATABASE: 'db'
      ME_CONFIG_OPTIONS_EDITORTHEME: 'ambiance'
      ME_CONFIG_BASICAUTH_USERNAME: 'user'
      ME_CONFIG_BASICAUTH_PASSWORD: 'password'
EOF
# Start docker-compose
docker-compose up -d --remove-orphans
# test using browser e.g. http://192.168.64.19:8081/ with user password
# after testing bring it down
docker-compose down -v

(16) Change password and install openssh-server
sudo apt install -y openssh-server
# change /etc/ssh/sshd_config PermitEmptyPasswords to yes
PermitEmptyPasswords yes

(17) Use another terminal session to connect e.g.
ssh-copy-id ubuntu@192.168.64.19
ssh ubuntu@192.168.64.19

(18) This iso image has desktop Applications, so it is possible to install vnc server and connect to it via Remote Desktop or vnc://localhost:590x. To install vnc server
sudo apt install -y tightvncserver
vncserver
# restart vncserver in vm
vncserver -localhost
# create ssh tunnel in Mac, where x is the desktop number when starting vncserver
ssh ubuntu@192.168.64.19 -L 590x:localhost:590x # Reference : https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-vnc-on-ubuntu-20-04
https://cat.pdx.edu/platforms/mac/remote-access/vnc-to-linux/

P.S. follow up post for the docker M1 preview and building of docker image with vnc server.
https://iphonesdkdev.blogspot.com/2020/12/20201205.html



If you need port forwarding on macOS to connect to this vm service (e.g. ssh or vnc), see this article https://www.papercut.com/kb/Main/MacPortForwarding