Thursday, April 8, 2021

How to write subrountine call in macOS assembly code for x86_64 and arm64

(1) Demo the subrountine call in macOS assembly code for x86_64 and arm64. For M1 Mac with Rosetta 2 installed, it can compile and run x86_64 and arm64 binary after installation of Xcode
callfactorial.c  Select all
/* * An application that illustrates calling the factorial function defined elsewhere. */ #include <stdio.h> #include <inttypes.h> uint64_t factorial(unsigned n); int main() { for (unsigned i = 0; i < 20; i++) { printf("factorial(%2u) = %llu\n", i, factorial(i)); } }

factorial.s  Select all
# ---------------------------------------------------------------------- # A 64-bit recursive implementation of the function # # uint64_t factorial(unsigned n) # # implemented recursively with x86_64 and arm64 assembly code # ----------------------------------------------------------------------- .globl _factorial .text #ifdef __arm64__ .align 4 #endif _factorial: #ifdef __x86_64__ cmp $1, %rdi # n <= 1? jnbe L1 # if not, go do a recursive call mov $1, %rax # otherwise return 1 ret #endif #ifdef __arm64__ cmp x8, #1 //# n > 1? L1 //# if yes, go do a recursive call mov x0, #1 //# otherwise return 1 ret #endif L1: #ifdef __x86_64__ push %rdi # save n on stack (also aligns %rsp!) dec %rdi # n-1 call _factorial # factorial(n-1), result goes in %rax pop %rdi # restore n imul %rdi, %rax # n * factorial(n-1), stored in %rax ret #endif #ifdef __arm64__ STP X8, LR, [SP, #-16]! //# push x8 and LR(x30) // LR is to return from subroutine subs x8, x8, #1 //# n-1 bl _factorial //# factorial(n-1), result goes in x0 LDP X8, LR, [SP], #16 //# pop x8 and LR(x30) mul x0, x0, x8 //# n * factorial(n-1), stored in x0 ret #endif

(2) To compile with -g and codesign the program so as to debug in lldb under macOS.
shell script  Select all
# To compile and codesign x86_64 version clang factorial.s callfactorial.c -g -o callfactorial_arm64 -arch x86_64 && codesign --entitlement entitlements --force -s - callfactorial_x86_64 # To compile and codesign arm64 version clang factorial.s callfactorial.c -g -o callfactorial_arm64 -arch arm64 && codesign --entitlement entitlements --force -s - callfactorial_arm64

(3) To debug using lldb
shell script  Select all
lldb callfactorial_x86_64 # or lldb callfactorial_arm64 # lldb debug session for arm64 - useful commands (lldb) breakpoint set --name main --name factorial (lldb) breakpoint list (lldb) run (lldb) step (lldb) po i (lldb) reg read x0 x8 lr pc (lldb) reg read -f t cpsr # lldb debug session for x86_64 - useful commands (lldb) reg read -f d rax rdi rflags (lldb) reg read -f t rflags # print the address value in the stackpointer for x86_64 (lldb) p *(int **)$sp # hint: to search lldb command history use ctrl-r

Saturday, April 3, 2021

How to use inline assembly language for M1 Mac for x86_64 and arm64

(1) Demo the inline assembly code for x86_64 and arm64
add-inline.c  Select all
#include <stdio.h> // compile with command line // clang add-inline.c -o add-inline_x86_64 -arch x86_64 // clang add-inline.c -o add-inline_arm64 -arch arm64 // disassemble with // otool -otV add-inline_x86_64 // otool -otV add-inline_arm64 int main(int argc, const char * argv[]) { int a = 10; int b = 25; int ans = 0; #ifdef __x86_64__ __asm__( "add %2,%1;\n" // %1 += %2 add source to destination "mov %1,%0;\n" // move data from %1 to %0 : "=r"(ans) : "r"(a), "r"(b) : ); #endif #ifdef __arm64__ __asm__( "add %w0,%w1,%w2;\n" // load %w0 = %w1 + %w2 : "=r"(ans) : "r"(a), "r"(b) : ); #endif printf("The answer is %d\n",ans); return 0; }

(2) Demo the assembly source code as a separate function for x86_64 and arm64
add-main.c  Select all
#include <stdio.h> // compile with command line // clang add-main.c add.s -o add-main_x86_64 -arch x86_64 // clang add-main.c add.s -o add-main_arm64 -arch arm64 #include <stdio.h> int add(int x, int y); int main(int argc, const char * argv[]) { int ans = add(15,40); printf("The answer is %d\n",ans); return 0; }

add.s  Select all
.text .globl _add .align 2 _add: #ifdef __x86_64__ add %esi,%edi // %edi += %esi, source is the first mov %edi,%eax // move data from %edi to %eax // x86_64 calling convention // rdi, rsi, rdx, rcx, r8, r9 // The 32-bit general purpose registers are edi, esi, edx, ecx, r8d, r9d instead. // The 16-bit general purpose registers are di, si, dx, cx, r8w, r9w instead. // The syscall number is placed in rax // see // Return value is in rax #endif #ifdef __arm64__ add w0,w0,w1 // load w0 with w0+w1, destination is the first // calling convention for arm64 // x0,x1,x2,x3,x4,x5,x6,x7 or r0 to r7 // The 32-bit general purpose registers are w0-w7 instead. // The syscall number is placed in r8 // Return value is in x0 // see #endif ret

entitlements  Select all
<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" ""> <plist version="1.0"> <dict> <key></key> <true/> </dict> </plist>

(3) Compile with
clang add-inline.c -o add-inline_x86_64 -arch x86_64
clang add-inline.c -o add-inline_arm64 -arch arm64
clang add-main.c add.s -o add-main_x86_64 -arch x86_64
clang add-main.c add.s -o add-main_arm64 -arch arm64

(4) Disassemble with (e.g.)
otool -otV add-inline_arm64

(5) Install Rosetta 2 on M1 to run the x86_64 version (e.g.)

(6) codesign and debug with lldb (e.g.)
clang -g -o add-inline_x86_64 add-inline.c -arch x86_64
codesign --entitlement entitlements --force -s - add-inline_x86_64
lldb add-inline_x86_64
(lldb) breakpoint set --file add-inline.c --line 7

(7) floating point example for x86_64 and arm64
compile with
clang sum.s callsum.c -o callsum_x86_64 -arch x86_64
clang sum.s callsum.c -o callsum_arm64 -arch arm64
Debug and codesign similar to above example
callsum.c  Select all
/* * callsum.c * * Illustrates how to call the sum.s function wrote in assembly language. */ // clang sum.s callsum.c -o callsum_x86_64 -arch x86_64 // clang sum.s callsum.c -o callsum_arm64 -arch arm64 #include <stdio.h> double sum(double[], unsigned); int main() { double test[] = { 40.5, 26.7, 21.9, 1.5, -40.5, -23.4 }; printf("%20.7f\n", sum(test, 6)); printf("%20.7f\n", sum(test, 2)); printf("%20.7f\n", sum(test, 0)); printf("%20.7f\n", sum(test, 3)); return 0; }

sum.s  Select all
# ----------------------------------------------------------------------- # A 64-bit function that returns the sum of the elements in a # floating-point array for x86_64 and arm64. The function has prototype: # # double sum(double[] array, unsigned length) # ----------------------------------------------------------------------- .global _sum .text .align 2 _sum: #ifdef __x86_64__ xorpd %xmm0, %xmm0 // initialize the sum to 0 // floats are passed in xmm0 cmp $0, %rsi // special case for length = 0 je done #endif #ifdef __arm64__ movi d0, #0 // initialize the sum to 0 // floats are passed in s0-7 and doubles in the d0-7 registers. cmp x1, #0 // special case for length = 0 b.eq done #endif next: #ifdef __x86_64__ addsd (%rdi), %xmm0 // add in the current array element, return floating point value in xmm0 add $8, %rdi // move to next array element dec %rsi // count down jnz next // if not done counting, continue #endif #ifdef __arm64__ ldr d16, [x0] // load the float into d16 // floats in s0-7 and doubles in the d0-7 registers. fadd d0, d0, d16 // add in the current array element, return floating point value in d0 add x0, x0, #8 // move to next array element subs x1, x1, #1 // count down cbnz w1, next // if not done counting, continue #endif done: ret

Wednesday, December 23, 2020

How to install Parallel Desktop and Windows 10 on Mac M1

Mac M1 can install Parallel Desktop Preview and run x64 Windows app

1) Install PD for M1 Mac from and click Try Technical Preview. Need to login and register.

Direct download link
MD5 checksum 5fc71f89fcc67ed13bac1995604a962f
Activation key You have to register and login to activate the product

2) Install Windows 10 Client ARM64 Insider Preview download from Need to login and register as Insider Preview Program.
Direct download link>
3) Update Win10 ARM to version 21277 which can install and run x64 Windows programs.
Have to login Windows Insider Program and set to "Dev Channel" to recieve preview updates.

Sunday, December 20, 2020

Buid docker image for M1 Mac with development tools and vnc

(1) With the release of docker preview for Mac M1, it is now possible to build some arm64 images.
Docker M1 preview

(2) Install the above preview and then use terminal the tools. The build script is based on Android SDK docker image for x64 and modify it for arm64. Moreover, swift 5.3.1, clang 10.0.0, kotlin 1.4.21, gradle 6.7.1, gcc 9.3.0, java 11, firefox, visual studio code and codeblocks are added for development purpose.

(3) Shell script run in Mac docker info # check if docker is installed properly in Mac M1
mkdir -p ${HOME}/android_sdk_vnc
cd ${HOME}/android_sdk_vnc
curl -O
curl -O
curl -O
curl -O
curl -O

(4) Create the following Dockfile
Dockerfile    Select all
cat >Dockerfile <<'HEREEOF' # ====================================================================== # # Android SDK Docker Image # ====================================================================== # # Base image # ---------------------------------------------------------------------- # FROM arm64v8/ubuntu:20.04 # install essential tools RUN apt-get update && \ apt-get dist-upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends git wget wget2 zip unzip && \ apt-get install -y build-essential vim # install Android SDK RUN DEBIAN_FRONTEND=noninteractive apt-get install -y android-sdk && \ rm -f /usr/lib/android-sdk/build-tools/27.0.1 # install Android SDK cmdline tools ARG ANDROID_SDK_VERSION=6858069 ENV ANDROID_SDK_ROOT /usr/lib/android-sdk RUN wget2 -q${ANDROID_SDK_VERSION} && \ unzip commandlinetools*linux*${ANDROID_SDK_VERSION}*.zip && \ mv cmdline-tools tools && \ mkdir -p ${ANDROID_SDK_ROOT}/cmdline-tools && \ mv tools ${ANDROID_SDK_ROOT}/cmdline-tools/ && \ (cd /usr/lib/android-sdk/cmdline-tools; ln -s tools latest) && \ rm -f commandlinetools*linux*.zip # download and install Gradle ARG GRADLE_VERSION=6.7.1 ARG GRADLE_DIST=bin RUN wget2 -q${GRADLE_VERSION}-${GRADLE_DIST}.zip && \ rm -fr /usr/share/gradle && \ unzip gradle*.zip -d /usr/share && \ rm gradle*.zip && \ mv /usr/share/gradle-${GRADLE_VERSION} /usr/share/gradle # download and install Kotlin compiler ARG KOTLIN_VERSION=1.4.10 RUN wget2 -q${KOTLIN_VERSION}/kotlin-compiler-${KOTLIN_VERSION}.zip && \ unzip *kotlin*.zip && \ mv kotlinc /usr/share && \ rm *kotlin*.zip # install swift5 compiler RUN wget -qO- | bash RUN apt install -y swiftlang # setup adb server EXPOSE 5037 # set the environment variables ARG JDK_VERSION=1.11.0 ENV JAVA_HOME /usr/lib/jvm/java-${JDK_VERSION}-openjdk-arm64 ENV GRADLE_HOME /usr/share/gradle ENV KOTLIN_HOME /usr/share/kotlinc ENV ANDROID_SDK_ROOT /usr/lib/android-sdk ENV ANDROID_HOME /usr/lib/android-sdk ENV ANDROID_TOOLS /usr/lib/android-sdk/tools ENV ANDROID_PLATFORMS /usr/lib/android-sdk/platforms ENV ANDROID_PLATFORM_TOOLS /usr/lib/android-sdk/platform-tools ENV ANDROID_BUILD_TOOLS /usr/lib/android-sdk/build-tools ENV ANDROID_CMDLINE_TOOLS /usr/lib/android-sdk/cmdline-tools/tools ENV PATH ${PATH}:${GRADLE_HOME}/bin:${KOTLIN_HOME}/bin:${ANDROID_BUILD_TOOLS}/debian:${ANDROID_TOOLS}/bin:${ANDROID_PLATFORM_TOOLS}:${ANDROID_CMDLINE_TOOLS}/bin:${ANDROID_SDK_ROOT}/emulator # install system image and accept licence, installation of emulator will fail for arm64 system RUN yes Y | ${ANDROID_CMDLINE_TOOLS}/bin/sdkmanager --install "platform-tools" "system-images;android-24;google_apis;arm64-v8a" "platforms;android-24" "build-tools;30.0.3" && \ wget && \ chmod +x adb && mv adb /usr/lib/android-sdk/platform-tools/ # install and configure SSH server EXPOSE 22 ADD sshd-banner /etc/ssh/ # ADD authorized_keys /tmp/ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server supervisor locales && \ mkdir -p /var/run/sshd /var/log/supervisord && \ locale-gen en en_US en_US.UTF-8 && \ apt-get remove -y locales && apt-get autoremove -y && \ FILE_SSHD_CONFIG="/etc/ssh/sshd_config" && \ echo "\nBanner /etc/ssh/sshd-banner" >> $FILE_SSHD_CONFIG && \ echo "\nPermitUserEnvironment=yes" >> $FILE_SSHD_CONFIG && \ ssh-keygen -q -N "" -f /root/.ssh/id_rsa && \ FILE_SSH_ENV="/root/.ssh/environment" && \ touch $FILE_SSH_ENV && chmod 600 $FILE_SSH_ENV && \ printenv | grep "JAVA_HOME\|GRADLE_HOME\|KOTLIN_HOME\|ANDROID_SDK_ROOT\|LD_LIBRARY_PATH\|PATH" >> $FILE_SSH_ENV && \ echo "\nauth required envfile=$FILE_SSH_ENV" >> /etc/pam.d/sshd && \ FILE_AUTH_KEYS="/root/.ssh/authorized_keys" && \ touch $FILE_AUTH_KEYS && chmod 600 $FILE_AUTH_KEYS && \ for file in /tmp/*.pub; \ do if [ -f "$file" ]; then echo "\n" >> $FILE_AUTH_KEYS && cat $file >> $FILE_AUTH_KEYS && echo "\n" >> $FILE_AUTH_KEYS; fi; \ done && \ (rm /tmp/*.pub 2> /dev/null || true) # install and configure VNC server ENV USER root ENV DISPLAY :1 EXPOSE 5901 ADD /tmp/ ADD /usr/local/bin/ ADD supervisord_vncserver.conf /etc/supervisor/conf.d/ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends xfce4 xfce4-goodies xfonts-base dbus-x11 tightvncserver expect && \ apt-get install -y --no-install-recommends firefox codeblocks && \ chmod +x /tmp/; sync && \ /tmp/ && \ rm /tmp/ && \ apt-get remove -y expect && apt-get autoremove -y && \ FILE_SSH_ENV="/root/.ssh/environment" && \ echo "DISPLAY=:1" >> $FILE_SSH_ENV # install and patch Visual Studio Code RUN apt update && apt install -y libsecret-1-0 libxss1 libgbm1 && apt-get clean RUN wget -O code_arm64.deb && dpkg -i code_arm64.deb && rm code_arm64.deb RUN sed 's/BIG-REQUESTS/_IG-REQUESTS/' /usr/lib/aarch64-linux-gnu/ | tee /usr/share/code/ > /dev/null ADD supervisord.conf /etc/supervisor/conf.d/ CMD ["/usr/bin/supervisord"] HEREEOF

(5) build the Dockfile
docker build -f Dockerfile -t arm64v8/android-sdk-vnc .

(6) Copy Mac host to authorized_keys and start the container
cp ~/.ssh/authorized_keys .
cat ~/.ssh/ >> authorized_keys

# To start the container as daemon
docker run -d --name android-sdk-vnc -p 5901:5901 -p 2222:22 -p 5037:5037 \
-v $(pwd)/authorized_keys:/root/.ssh/authorized_keys arm64v8/android-sdk-vnc

# To connect the container via ssh
ssh root@<IP address of your Mac M1> -p 2222

# To stop the container
docker container stop android-sdk-vnc

(7) To connect to the vnc of the docker image after starting the container
Use the Mac Finder "Go" -> "Connect To Server" and enter "vnc://"
For other machines in the network enter "vnc://<IP address of your Mac M1>:5901"

The passwords are in the when building which are

(8) There is a common problem if you are connecting remotely using Mac Terminal, so the solution is to not forward your locale. Edit /etc/ssh/ssh_config and comment out SendEnv LANG LC_* line.

(9) Testing swift
# attach to the running container
docker exec -it android-sdk-vnc bash
# test
mkdir -p ${HOME}/Projects
cd ${HOME}/Projects
git clone
cd example-package-dealer
swift run Dealer

Saturday, December 19, 2020

How to setup debian vm with VNC on New Mac Mini M1

(1) Download the ubuntu kernel and images
cd ${HOME}
mkdir debianvm
curl -o - | gunzip > debianvm/vmlinuz
curl -o debianvm/initrd
curl -o - | tar zxvC debianvm

(2) download the raspbian arm64 image
curl -OL
unzip -d debianvm

(3) Copy and expand the disk to say 38G (Don't use this method to expand disk on external HD)
cp debianvm/focal-server-cloudimg-arm64.img debianvm/raspberry-arm64.img
dd if=/dev/zero of=debianvm/raspberry-arm64.img seek=40000000 obs=1024 count=0

(4) Compile and use the latest vftool which is enhanced to mount multiple disks.
cd ${HOME}
git clone
cd vftool && xcodebuild

(5) start the vm say
screen -S vm
cd ${HOME}/debianvm
${HOME}/vftool/build/Release/vftool -k vmlinuz -i initrd -d focal-server-cloudimg-arm64.img -d raspberry-arm64.img -d 2020-08-20-raspios-buster-arm64.img -m 2048 -a "console=hvc0"
Detach the screen using Ctrl-A D

(6) start the tty say
screen -S tty /dev/ttys001

(7) prepare the ubuntu cloud image
mkdir /mnt
mount /dev/vda /mnt
chroot /mnt

touch /etc/cloud/cloud-init.disabled

echo 'root:root' | chpasswd

ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N '' -t rsa
ssh-keygen -f /etc/ssh/ssh_host_dsa_key -N '' -t dsa
ssh-keygen -f /etc/ssh/ssh_host_ecdsa_key -N '' -t ecdsa
ssh-keygen -f /etc/ssh/ssh_host_ed25519_key -N '' -t ed25519

# add fixed IP
cat <<EOF > /etc/netplan/01-dhcp.yaml 
    renderer: networkd
            dhcp4: true
            addresses: []
    version: 2

# add IP when login prompt
cat <<EOF >> /etc/issue
enp0s1: \4


umount /dev/vda

(8) copy the rootfs to raspberry-arm64.img
#Detach the screen using Ctrl-A D
screen -r vm # resume vm screen # stop the vm by ctrl-C and restart the ubuntu vm
cd ${HOME}/debianvm
${HOME}/vftool/build/Release/vftool -k vmlinuz -i initrd -d focal-server-cloudimg-arm64.img -d raspberry-arm64.img -d 2020-08-20-raspios-buster-arm64.img -m 2048 -a "console=hvc0 root=/dev/vda"

#Detach the screen using Ctrl-A D
#Notice the tty number when starting the vm and start the tty say
screen -S tty /dev/ttys001

# login with id root and password root
# and copy the rootfs to the expanded disk raspberry-arm64.img
dd if=/dev/vdc2 of=/dev/vdb bs=64K conv=noerror,sync

(9) prepare the raspberry-arm64 image
# stop the vm and restart the vm
# Control-A D to detach the tty screen # resume vm screen screen -r vm # Control-C to stop the vm cd ${HOME}/debianvm
${HOME}/vftool/build/Release/vftool -k vmlinuz -i initrd -d raspberry-arm64.img -m 2048 -a "console=hvc0"
mkdir /mnt
mount /dev/vda /mnt
chroot /mnt

echo 'root:root' | chpasswd

#!!! delete or comment out the lines in /etc/fstab, or there will be "A start job is running on /dev/disk/.." blocking!!
cat <<EOF > /etc/fstab
proc            /proc           proc    defaults          0       0
#PARTUUID=ad09722e-01  /boot           vfat    defaults          0       2
#PARTUUID=ad09722e-02  /               ext4    defaults,noatime  0       1

ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N '' -t rsa
ssh-keygen -f /etc/ssh/ssh_host_dsa_key -N '' -t dsa
ssh-keygen -f /etc/ssh/ssh_host_ecdsa_key -N '' -t ecdsa
ssh-keygen -f /etc/ssh/ssh_host_ed25519_key -N '' -t ed25519

mkdir -p /etc/network/interfaces.d/
cat <<EOF > /etc/network/interfaces.d/enp0s1 
allow-hotplug enp0s1
iface enp0s1 inet dhcp
auto lo
iface lo inet loopback

cat <<EOF >> /etc/dhcpcd.conf
# add fixed IP
interface enp0s1
static ip_address=

# put mac host ${HOME}/.ssh/ key here
mkdir -p /root/.ssh
cat <<EOF > /root/.ssh/authorized_keys
ssh-rsa <mac key here>

# add IP when login prompt
cat <<EOF >> /etc/issue
enp0s1: \4


umount /dev/vda

(10) login raspberry OS arm64 with id root and password root
# stop the vm and restart the vm with
cd ${HOME}/debianvm
${HOME}/vftool/build/Release/vftool -k vmlinuz -i initrd -d raspberry-arm64.img -m 2048 -a "console=hvc0 root=/dev/vda rw"
# set date say
date -s "29 DEC 2020 14:54:00"
resize2fs /dev/vda
dpkg-reconfigure tzdata # set timezone
apt update
systemctl disable rpi-eeprom-update.service
systemctl disable rng-tools.service

(11) install utilities in raspberry os that can handle qcow2 disk image conversion and resize
apt install qemu-utils libguestfs-tools
hostname -I # get ip address

(13) Install Realvncserver and connect from Mac
passwd pi # set user pi password
# login as user pi
login pi
# Use raspi-config to enable SSH and VNC
sudo apt install realvnc-vnc-server
sudo raspi-config
# Choose 5 Interfacing Options -> P2 SSH
# Choose 5 Interfacing Options -> P3 VNC

# enter vnc key, can put this in /etc/rc.local without sudo
sudo vnclicense -add VKUPN-MTHHC-UDHGS-UWD76-6N36A

# setup vncpasswd for user mode, only user mode can run browser
vncpasswd -user

# edit config file to have these two lines for vnc password auth
echo -e "Authentication=VncAuth" | tee -a ~/.vnc/config.d/vncserver-x11
echo -e "Encryption=PreferOff" | tee -a ~/.vnc/config.d/vncserver-x11
cp ~/.vnc/config.d/vncserver-x11 ~/.vnc/config.d/Xvnc
# start service

#put these in rc.local to automate after reboot, Raspberry Pi hardware has bundled vnclicense and no need to add it
/usr/bin/vnclicense -add VKUPN-MTHHC-UDHGS-UWD76-6N36A
sudo -H -u pi /usr/bin/vncserver

#check which port is listening
netstat -tlpn | grep -i listen

This would configure VNC in Virtual Mode
# Screen Sharing Connect from Mac
# In Finder, Go -> Connect to Server, enter IP address and port number of the vm e.g.
# Enter the vncpassword to connect, and connection needs port number

For configuring VNC in Service Mode, please refer to
For differences in these modes, please refer to
#Start or stop the service with:
# systemctl (start|stop) vncserver-x11-serviced.service
#Mark or unmark the service to be started at boot time with:
# systemctl (enable|disable) vncserver-x11-serviced.service
#sudo systemctl start vncserver-x11-serviced.service
For Service Mode the connection from Mac is (no port number), vnc://

# To install Visual Studio Code
wget -O code_arm64.deb && sudo dpkg -i code_arm64.deb && rm code_arm64.deb
sed 's/BIG-REQUESTS/_IG-REQUESTS/' /usr/lib/aarch64-linux-gnu/ | sudo tee /usr/share/code/ > /dev/null

Demo - VNC to rapberrypi desktop vm screen capture

P.S. Only vm can run realvncserver, docker environment can only run lightweight vncserver e.g. tightvncserver

(14) For qcow2 image, the conversion to raw image is
qemu-img dd -f qcow2 -O raw bs=64k if=debian-10-generic-arm64-20201214-484.qcow2 of=debian-10.img
# scp debian-10.img to the Mac Mini host and prepare a new expanded debian-arm64.img similar to the raspberry-arm64.img as above
# After restarting vm and mounting images, the dd command to extract rootfs is
# Assume debian-arm64.img is vdb and debian-10.img is vdc
dd if=/dev/vdc1 of=/dev/vdb bs=64K conv=noerror,sync
# For debian image, add this to /etc/hosts debian

Thursday, December 3, 2020

How to run docker on the new Mac Mini M1

(1) Reference to this article

(2) The problem of this method of running Ubuntu iso live image is not persistent.

(3) So here is the improved method to add persistent storage to this vm.

(4) # First create an empty image data say 50G in Mac Mini
dd if=/dev/zero of=data.img bs=1m count=51200

(5) # Change the command to invoke the vm to include persistent and increase cpu to 2
./vftool -k vmlinuz -i initrd -c focal-desktop-arm64.iso -d data.img -p 2 -m 4096 -a "console=hvc0 persistent rw"
./vftool -k vmlinuz -i initrd -d data.img -c focal-desktop-arm64.iso -p 2 -m 4096 -a "console=hvc0 persistent rw"

If you compile the latest vftool, which has added the ability to add more image files and the order of the mounted device will follow the parameters you passed in. The article was written using the first version of the vftool and will mount the data.img as the /dev/vda and the cdrom image as /dev/vdb. So if you use the latest version vftool compiled, you have to check the device name of your mounted data.img.

(6) # Attach the vm and create 3 partitions for the created image
sudo fdisk /dev/vda
# Press (n)(p)(1) to create new partition 1
# Press (enter) to accept starting sector and set size to say +25G
# Press (n)(p)(2) to create new partition 2
# Press (enter) to accept starting sector and set size to say +15G
# Press (n)(p)(3) to create new partition 3
# Accept the default options and starting block and ending block
# Press (w) to write to partition

(6) # Format partition as ext4 with labels casper-rw and home-rw in vm, this create the persistent medium for the rw partition.
sudo mkfs.ext4 /dev/vda1 -L docker-data
sudo mkfs.ext4 /dev/vda2 -L casper-rw
sudo mkfs.ext4 /dev/vda3 -L home-rw

(7) # exit shell and exit vm by ctrl-c

(8) # start vm again and use a new tab using screen command to attach to the vm e.g.
screen -S docker /dev/ttys003

(9) # Create folder to mount /dev/vda1
sudo mkdir -p /mnt/docker-data
# Add this mount commands in /etc/rc.local in order to persist as editing /etc/fstab is ignored in this iso image
cat << EOF | sudo tee -a /etc/rc.local 
#!/bin/sh -e
# rc.local
# This script is executed at the end of each multiuser runlevel.
# Make sure that the script will "exit 0" on success or any other
# value on error.
# In order to enable or disable this script just change the execution
# bits.
# By default this script does nothing.

mount /dev/vda1 -t ext4 /mnt/docker-data
exit 0

cat << EOF | sudo tee -a /etc/systemd/system/rc-local.service
 Description=/etc/rc.local Compatibility

 ExecStart=/etc/rc.local start


# enable rc.local see this.
sudo chmod +x /etc/rc.local
sudo systemctl start rc-local.service
sudo systemctl status rc-local.service

(10) Follow the reference article to install docker related packages.
If you have problem when installing docker. Try this = previous verion
apt install docker-ce=5:19.03.14~3-0~ubuntu-focal

(11) # Move docker data-root to the newly created partition
sudo service docker stop
cat << EOF | sudo tee -a /etc/docker/daemon.json 
"storage-driver": "overlay2",
"data-root": "/mnt/docker-data"
sudo rsync -aP /var/lib/docker/ /mnt/docker-data
sudo mv /var/lib/docker /var/lib/docker.old
sudo service docker start

(12) Additionally, install docker-compose
sudo add-apt-repository universe
sudo apt install -y python3-pip
sudo pip3 -v install docker-compose

(13) Test docker
docker run hello-world

docker run -e MYSQL_ROOT_PASSWORD=rootpassword -e MYSQL_USER=wpuser -e MYSQL_PASSWORD=wpuserpassword -e MYSQL_DATABASE=wordpressdb --name wordpressdb -d arm64v8/mariadb
docker run -e WORDPRESS_DB_USER=wpuser -e WORDPRESS_DB_PASSWORD=wpuserpassword -e WORDPRESS_DB_NAME=wordpressdb -p 8080:80 --link wordpressdb:mysql --name wordpress -d arm64v8/wordpress
# show IP address
ip addr | grep inet
# test using browser e.g.

(14) Test docker-compose 1 (wordpress and mysql)
# create docker-compose.yaml
cd $HOME
mkdir my-wordpress
cd my-wordpress
cat > docker-compose.yaml <<EOF
version: '3'
    image: arm64v8/mariadb
      - "./.data/db:/var/lib/mysql"
    restart: always
      MYSQL_ROOT_PASSWORD: rootpassword
      MYSQL_DATABASE: wordpressdb
      MYSQL_USER: wpuser
      MYSQL_PASSWORD: wpuserpassword

      - db
    image: arm64v8/wordpress
      - db:mysql
      - "8000:80"
    restart: always
      WORDPRESS_DB_HOST: db:3306
      WORDPRESS_DB_NAME: wordpressdb
      WORDPRESS_DB_USER: wpuser
      WORDPRESS_DB_PASSWORD: wpuserpassword
# Start docker-compose
docker-compose up -d
# test using browser e.g.
# after testing bring it down
docker-compose down -v

(15) Test docker-compose 2 (mongo-express)
# create docker-compose.yaml
cd $HOME
mkdir my-mongoexpress
cd my-mongoexpress
cat > docker-compose.yaml <<EOF
version: '3.1'
    image: arm64v8/mongo
      - ./data:/data/db
      - '27018:27017'
    restart: always

    image: arm64v8/mongo-express
      - mongo
      - "8081:8081"
    restart: always
# Start docker-compose
docker-compose up -d --remove-orphans
# test using browser e.g. with user password
# after testing bring it down
docker-compose down -v

(16) Change password and install openssh-server
sudo apt install -y openssh-server
# change /etc/ssh/sshd_config PermitEmptyPasswords to yes
PermitEmptyPasswords yes

(17) Use another terminal session to connect e.g.
ssh-copy-id ubuntu@
ssh ubuntu@

(18) This iso image has desktop Applications, so it is possible to install vnc server and connect to it via Remote Desktop or vnc://localhost:590x. To install vnc server
sudo apt install -y tightvncserver
# restart vncserver in vm
vncserver -localhost
# create ssh tunnel in Mac, where x is the desktop number when starting vncserver
ssh ubuntu@ -L 590x:localhost:590x # Reference :

P.S. follow up post for the docker M1 preview and building of docker image with vnc server.

How to run on Headless Mac Mini M1 with nohup and screen

(1) Need a HDMI dummy simulator attached to Mac Mini for full speed operation. Be sure to have HDMI 2.0 and 4K support and some dummy simulators have Pass-Through EDID Emulator (also with 4K support) so that you can attach 4K monitor or via screen sharing in HD when working on it with attached keyboard and mouse.

(2) Disable sleep mode and set never sleep for HD and monitor on System Preferences of Mac Mini.

(3) Enable Sharing and SSH login on Mac Mini.

(4) generate ssh key-pair on mac mini and your terminal or your phone. If using Android phone, recommend Termux app for remote shell access.
e.g. ssh-keygen -t rsa # generate key pair on Mac Mini and remote client

(5) copy the ssh key-pair to mac mini if using another terminal from mac.
e.g. ssh-copy-id user@x.x.x.x

(6) Login in Mac Mini via ssh remotely
e.g. ssh user@x.x.x.x

(7) # Apple Compressor command line batch submission example (no need for nohup)
/Applications/ -batchname "My First Batch" \
-jobpath ~/Movies/MySource_x265.mp4 \
-settingpath /Applications/ \
-locationpath ~/Movies/

(8) # install youtube-dl, prerequisite: Command Line Tools for Xcode 12.2 for Python 3.8 to be installed
sudo mkdir -p /usr/local/bin
sudo curl -L -o /usr/local/bin/youtube-dl
sudo chmod a+rx /usr/local/bin/youtube-dl
youtube-dl --version
sudo youtube-dl -U # update youtube-dl
youtube-dl --version # check version after update
cd $HOME/Downloads
mkdir -p youtube
cd youtube
youtube-dl -f mp4 <youtube url>

(9) Either use nohup or screen command for persistent jobs.

# nohup example
nohup bash -c 'youtube-dl -f 401' &> nohup.youtubedl.4k_2.out &

(10)screen example (suitable for command utility with tty input
# create a screen and run python script
screen -S tensorflow
conda activate python38
# leave (detach) the screen by Control-A d

# create second screen and run handbrake conversion
screen -S handbrake
conda deactivate
$HOME/Downloads/HandBrakeCLI -i $HOME/Downloads/8KVIDEO120FPS-8rPB4A3zDnQ_4.1G.mp4 -o $HOME/Downloads/8KVIDEO_x265.mp4 -e x265 -q24
# leave (detach) the screen by Control-A d

# list all the screen ids
screen -ls

# Reattach to one of the screen e.g. 6473.tensorflow
screen -r 6473

# Exit Screen if the process terimated