ARM_PATH should be /usr! Various other improvements.

README.md

+## magic-tools - tools for Magic Lantern!
+
+Just some utilities for prepping CF cards, compiling post tools, compiling ML itself, and managing the source code.
+
+
+## Cool Functions
+Things we can do.
+
+* Prepare a Magic Lantern SD/CF card: Run `./formatML.sh` (with options; see -h option) in the **prep_card** folder.
+* Auto build ML binary: Run `./mkML.sh` in the **bin** folder.
+* Build post proc tools: Run `./mkToolStack.sh' and potentially `./mkDcraw.sh` in the **tools** folder.
+
+
+## How to make it work
+Before doing anything, run: `./prepAll.sh`. This will compile stuff & get the ML source code.
+
+**For Debian/Ubuntu:**
+
+Make sure to install the ARM toolchain: `sudo apt-get install gcc-arm-none-eabi` (ARM_PATH is already set correctly to /usr).
+
+**If you are not on Debian/Ubuntu:**
+
+1. Get gcc version 5.4.1 or similar.
+2. Enter src. Run `./getArm.sh`. 
+3. Edit src/patches/armpath.patch: On the "ARM_PATH" line beggining with "+", replace "/usr" with "~/gcc-arm-none-eabi-5_4-2016q3"
+
+*Other gcc versions: In the patch file, put in your gcc version and corresponding working gcc-arm-none-eabi path (Browse https://launchpad.net/gcc-arm-embedded for that)*

bin/mkML.sh

+#!/bin/bash
+
+SCRIPTPATH=$( cd $(dirname $0) ; pwd -P )
+ML_PATH="${SCRIPTPATH}/../src/magic-lantern/platform/7D.203"
+
+rm -rf $SCRIPTPATH/stable
+mv $SCRIPTPATH/latest $SCRIPTPATH/stable
+
+cd $ML_PATH
+
+#Prepare source.
+cd $ML_PATH/../..
+patch < ../patches/* #Apply patches.
+make clean 
+cd $ML_PATH
+
+#Do the compile.
+make clean; make -j; make zip
+
+files=( "*.zip" )
+echo "Built ${files[0]}."
+unzip "${files[0]}" -d build #unzip the created zip file into "build".
+
+cp -r build $SCRIPTPATH/latest
+rm -rf build
+
+#Cleanup
+cd $ML_PATH/../..
+patch -R < ../patches/* #Remove patches.

prepAll.sh

+#!/bin/bash
+
+SCRIPTPATH=$( cd $(dirname $0) ; pwd -P )
+
+#Compile exfat_sum.
+cd ${SCRIPTPATH}/prep_card
+gcc exfat_sum.c -o exfat_sum
+
+#Get Magic Lantern source
+cd ${SCRIPTPATH}/src
+./getSrc.sh
+
+#All Done!

prep_card/exfat_sum.c

+/*
+ * recompute a exFAT VBR checksum in sector 11 (first is 0)
+ */
+
+#include<stdio.h>
+
+typedef unsigned long UINT32;
+
+UINT32 VBRChecksum( unsigned char octets[], long NumberOfBytes) {
+   UINT32 Checksum = 0;
+   long Index;
+   for (Index = 0; Index < NumberOfBytes; Index++) {
+     if (Index != 106 && Index != 107 && Index != 112)  // skip 'volume flags' and 'percent in use'
+	 Checksum = ((Checksum <<31) | (Checksum>> 1)) + (UINT32) octets[Index];
+   }
+   return Checksum;
+}
+
+inline unsigned long endian_swap(unsigned long x) {
+   return (x<<24) | ((x>>8) & 0x0000FF00) | ((x<<8) & 0x00FF0000) | (x>>24);
+}
+
+#define SIZE (512*11)
+
+int main(int argc, char *argv[]) {
+  FILE* f;
+  unsigned long buffer[SIZE+512];
+  int i=0;
+  unsigned long sum, swappedSum;
+
+  f=(FILE*)fopen(argv[1], "rb+");
+  if (f) {
+    fread(buffer, 1, SIZE+512, f);
+    printf("old=0x%lx, ", buffer[ SIZE/4 ]);
+    sum = VBRChecksum((unsigned char*)buffer, SIZE);
+    printf("new=0x%lx\n", sum);
+//    swappedSum = endian_swap( sum );
+    //    printf("0x%lx\n", swappedSum);
+    for(i=0; i<512/4; i++)
+      buffer[ i] = sum;  // works only in Litte Endian architectures
+    fseek(f, SIZE, SEEK_SET);
+    fwrite(buffer, 1, 512, f);
+    fclose(f);
+  }
+
+}

prep_card/formatML.sh

+#!/bin/bash
+
+#Run with sudo. Always.
+#To use compile, you must have setup the source code to be able to compile with make already!
+
+SCRIPTPATH=$( cd $(dirname $0) ; pwd -P )
+
+#Valid options: src, bin.
+ML="bin"
+ML_PATH="" #Means different things based on the value of ML.
+SD_NAME="EOS_DIGITAL"
+
+help() {
+less -R << EOF 
+Usage:
+	$(echo -e "\033[1m./formatML.sh\033[0m -r <path> [OPTIONS] /dev/sd<??>")
+
+PATTERN GUIDELINES:
+	Format a Magic Lantern flashcard, potentially compiling Magic Lantern to do so.
+	
+	You must place make_bootable.sh and exfat_sum in the same directory.
+	
+	Download the "cleaned" source code: https://groups.google.com/forum/#!topic/ml-devel/EWHU5ukyMt4 .
+		*Compile exfat_sum with 'gcc exfat_sum.c -o exfat_sum'.
+		
+	To do an auto source compile, it must already be setup to compile correctly with make (the compiler toolchain must be working correctly).
+
+	Ex. sudo ./formatML.sh -p ~/magic-lantern/bin/latest /dev/sdc1
+	Ex. sudo ./formatML.sh -m src -p ~/subhome/src/magic-lantern/platform/7D.203 /dev/sdc1
+
+OPTIONS:
+	-m		ml_src - Where to get ML from. 'src' will compile a nightly build. 'bin' will copy prebuilt files over.
+				*bin is default.
+	-p		path - If compiling, path to the platform in the source. If using binary, path to the ML build.
+				*Defaults to nothing - must be specified.
+	-n		name - The name of the prepared SD card. Must match length of EOS_DIGITAL
+
+EOF
+}
+
+while getopts "vh m: p: n:" opt; do
+	case "$opt" in
+		h)
+			help
+			exit 0
+			;;		
+		
+		v)
+			echo -e "See formatML.sh -h"
+			exit 0
+			;;
+		m)
+			ML=${OPTARG}
+			;;
+		p)
+			ML_PATH=$(readlink -f ${OPTARG})
+			if [[ ! -d $ML_PATH ]]; then
+				echo "Source path does not exist!"
+			fi
+			;;
+		n)
+			SD_NAME=${OPTARG}
+			;;
+		*)
+			echo "Invalid option: -$OPTARG" >&2
+			;;
+	esac
+done
+
+shift $((OPTIND-1))
+SD_LOC=$1
+
+if [ -z $SD_LOC ]; then echo "Not a valid device!"; exit 1; fi
+if [ -z $ML_PATH ]; then echo "Not a valid path!"; exit 1; fi
+
+#Run some checks. These are dangerous operations :).
+
+#Format filesystem.
+umount "$SD_LOC" 2>/dev/null
+mkdosfs -F 32 -I "$SD_LOC"
+
+copyML() {
+	ML_BIN="$1"
+	MOUNTPOINT="/media/$(logname)/$SD_NAME"
+	
+	mount "$SD_LOC" $MOUNTPOINT
+	cp -r $ML_BIN/* $MOUNTPOINT
+	sleep 0.5
+	umount "$SD_LOC"
+	sleep 0.5
+}
+
+#Potentially compile & copy files over.
+if [[ $ML == "src" ]]; then
+	#Prepare source.
+	cd $ML_PATH/../..
+	patch < ../patches/* #Apply patches.
+	
+	cd $ML_PATH
+	
+	#Have to build ML as the user who is sudoing in.
+	su $(logname) -c "make clean; make -j; make zip"
+	
+	files=( "*.zip" )
+	echo "Built ${files[0]}."
+	unzip "${files[0]}" -d build #unzip the created zip file into "build".
+	
+	copyML "$(pwd)/build"
+	
+	#Cleanup
+	rm -rf build
+	su $(logname) -c "make clean"
+	
+	#Clean Patches
+	cd $ML_PATH/../..
+	patch -R < ../patches/* #Remove patches.
+	
+	cd $SCRIPTPATH
+elif [[ $ML == "bin" ]]; then
+	copyML "$ML_PATH"
+fi
+
+#Make bootable with make_bootable.sh & eject.
+$SCRIPTPATH/make_bootable.sh "$SD_LOC" $SD_NAME
+#~ python3 "$SD_LOC" "$SD_NAME"
+eject "$SD_LOC"
+
+echo -e "\nFinished treating card! You can safely eject now."

prep_card/make_bootable.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2013 Diego Elio Pettenò <flameeyes@flameeyes.eu>
+# Inspired by Trammel's and arm.indiana@gmail.com's shell script
+# 
+#         DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE 
+#                     Version 2, December 2004 
+# 
+#  Copyright (C) 2004 Sam Hocevar <sam@hocevar.net> 
+# 
+#  Everyone is permitted to copy and distribute verbatim or modified 
+#  copies of this license document, and changing it is allowed as long 
+#  as the name is changed. 
+# 
+#             DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE 
+#    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 
+# 
+#   0. You just DO WHAT THE FUCK YOU WANT TO.
+# 
+#
+# patch the SD/CF card bootsector to make it bootable on Canon DSLR
+# See http://chdk.setepontos.com/index.php/topic,4214.0.html
+#
+# usage: make_bootable.py device (e.g. /dev/sdb1)
+#
+# Supports FAT16, FAT32 and EXFAT devices; should work on all
+# operating systems as long as Numpy is available.
+
+import platform
+import sys
+import mmap
+import struct
+
+from numpy import uint8, uint32
+
+def setboot(device, of1, of2):
+    print('Applying parameters on device')
+    print('  writing %s at offset %x (Volume Label)' % (sys.argv[2].strip(), of1) )
+    print('  writing BOOTDISK at offset %x (Boot code)' % of2)
+    device[of1:of1+11] = bytes(sys.argv[2].strip(), "UTF-8")
+    device[of2:of2+8] = b'BOOTDISK'
+
+def setExfat(device, offset):
+    print('Updating EXFAT VBR at %x' % offset)
+    setboot(device, offset + 130, offset + 122)
+
+    EXFAT_VBR_SIZE = 512 * 11
+
+    print('Calculating VBR checksum')
+    checksum = uint32(0)
+    for index in range(0, EXFAT_VBR_SIZE):
+        if index in {106, 107, 112}:
+            continue
+        value = uint8(device[offset+index])
+        checksum = uint32((checksum << 31) | (checksum >> 1) + value)
+
+    checksum_chunk = struct.pack('<I', checksum) * 128
+    device[offset+512*11:offset+512*12] = checksum_chunk
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
+
+    device_fd = open(argv[1].strip(), 'r+b')
+    # Only map the longest boot recod we need to write to (512*24 is
+    # needed for exfat)
+    device = mmap.mmap(device_fd.fileno(), 512*24)
+
+    if device[54:62] == b'FAT16   ':
+        print('Identified FAT16 partition')
+        setboot(device, 43, 64)
+    elif device[82:90] == b'FAT32   ':
+        print('Identified FAT32 partition')
+        setboot(device, 71, 92)
+    elif device[3:11] == b'EXFAT   ':
+        print('Identified EXFAT partition')
+        setExfat(device, 0)
+        # backup VBR, set that one up as well
+        setExfat(device, 512*12)
+    else:
+        print("""Device %s is not a valid filesystem in the supported range.
+Supported filesystems are FAT16, FAT32, EXFAT.
+Format your card in camera before using this script.""" % argv[1],
+              file=sys.stderr
+	  )
+
+    device.flush()
+    device_fd.close()
+
+if __name__ == '__main__':
+    main()

prep_card/make_bootable.sh

+#!/bin/bash
+
+# v1 : FAT16 and FAT32, based on Trammel version
+# v2 : exFAT supported. 
+# arm.indiana@gmail.com
+#
+# patch the SD/CF card bootsector to make it bootable on Canon DSLR
+# See http://chdk.setepontos.com/index.php/topic,4214.0.html
+#     http://en.wikipedia.org/wiki/File_Allocation_Table#Boot_Sector
+#     http://www.sans.org/reading_room/whitepapers/forensics/reverse-engineering-microsoft-exfat-file-system_33274
+
+# usage: make_bootable.sh <device> (/dev/sdc1 for example) <name> (EOS_STABLE for example).
+# exfat_sum.c must bu compiled first
+
+dev=$1
+name=$2
+dump_file=exfat_dump.bin
+
+# read the boot sector to determine the filesystem version
+DEV64=`dd if=$dev bs=1 skip=3 count=8 2>/dev/null` 
+DEV32=`dd if="$dev" bs=1 skip=82 count=8 2>/dev/null`
+DEV16=`dd if="$dev" bs=1 skip=54 count=8 2>/dev/null`
+if [ "$DEV16" != 'FAT16   ' -a "$DEV32" != 'FAT32   ' -a "$DEV64" != 'EXFAT   ' ]; then
+  echo "Error: "$dev" is not a FAT16 or FAT32 device"
+  exit
+fi
+if [ "$DEV16" = 'FAT16   ' ]; then
+  offset1=43
+  offset2=64
+  FS='FAT16'
+elif [ "$DEV32" = 'FAT32   ' ]; then
+  offset1=71
+  offset2=92
+  FS='FAT32'
+elif [ "$DEV64" = 'EXFAT   ' ]; then
+  offset1=130
+  offset2=122
+  FS='EXFAT'
+else
+  echo "Error: "$dev" is not a FAT16, FAT32 or ExFAT (FAT64) device"
+  exit
+fi
+echo "Applying "$FS" parameters on "$dev" device:"
+echo " writing $name at offset" $offset1 
+echo $name | dd of="$dev" bs=1 seek=$offset1 count=11 2>/dev/null
+echo " writing BOOTDISK at offset" $offset2 
+echo BOOTDISK | dd of="$dev" bs=1 seek=$offset2 count=8 2>/dev/null
+if [ "$FS" = 'EXFAT' ]; then
+  # write them also in backup VBR, at sector 13
+  echo $name | dd of="$dev" bs=1 seek=$(($offset1++512*12)) count=11 2>/dev/null
+  echo BOOTDISK | dd of="$dev" bs=1 seek=$(($offset2+512*12)) count=8 2>/dev/null
+  dd if=$dev of="$dump_file" bs=1 skip=0 count=6144 2>/dev/null
+  echo -n ' recompute checksum. '
+  ./exfat_sum "$dump_file" 
+  # write VBR checksum (from sector 0 to sector 10) at offset 5632 (sector 11) and offset 11776 (sector 23, for backup VBR)
+  # checksum sector is stored in $dump_file at offset 5632 
+  dd of="$dev" if="$dump_file" bs=1 seek=5632 skip=5632 count=512 2>/dev/null
+  dd of="$dev" if="$dump_file" bs=1 seek=11776 skip=5632 count=512 2>/dev/null
+  #dd if=$dev of=verif_dump.bin bs=1 skip=0 count=12288 2>/dev/null
+  rm -f "$dump_file"
+fi
+

src/getArm.sh

+#!/bin/bash
+
+#Assumes gcc 5.4.X !!
+
+pkgName=gcc-arm-none-eabi-5_4-2016q3 #Get from launchpas version
+
+if [[ -d ~/${pkgName} ]]; then echo "Found ARM. Not downloading new one."; exit 0; fi
+
+wget https://launchpad.net/gcc-arm-embedded/5.0/5-2016-q3-update/+download/gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 -O ~/${pkgName}.tar.bz2
+
+mkdir -p ~/${pkgName}
+tar -C ~/${pkgName} -xvjf ~/${pkgName}.tar.bz2
+rm ~/${pkgName}.tar.bz2
+
+echo -e "\nMake sure to check patches/armpath.patch!"

src/getSrc.sh

+#!/bin/bash
+
+#Get HG stuff.
+hg clone https://bitbucket.org/hudson/magic-lantern
+cd magic-lantern
+hg checkout unified
+hg pull

src/patches/armpath.patch

+diff -r 4f72a0eabb16 Makefile.user.default
+--- a/Makefile.user.default	Sun Dec 18 15:08:37 2016 +0200
++++ b/Makefile.user.default	Fri Dec 30 02:09:10 2016 +0100
+@@ -93,13 +93,13 @@
+ # GCC-related stuff
+ # for yagarto or the official ARM toolchain use ARM_ABI=none-eabi otherwise use ARM_ABI=elf
+ ARM_ABI=none-eabi
+-GCC_VERSION=-4.8.3
++GCC_VERSION=-5.4.1
+ DIETLIB_VERSION=0.33
+ 
+ NEWLIB_OPT_LEVEL=O3
+ DIETLIBC_OPT_LEVEL=Os
+ 
+-ARM_PATH=~/gcc-arm-none-eabi-4_8-2013q4
++ARM_PATH=/usr
+ ARM_BINPATH=$(ARM_PATH)/bin
+ ARM_LIBGCC_PATH=$(ARM_PATH)/lib/gcc/arm-$(ARM_ABI)/$(subst -,,$(GCC_VERSION))
+ NEWLIB_PATH=$(SRC_DIR)/libs/arm-$(ARM_ABI)-$(NEWLIB_OPT_LEVEL)

tools/mkDcraw.sh

+#!/bin/bash
+
+SCRIPTPATH=$( cd $(dirname $0) ; pwd -P )
+TOOLLOC=$SCRIPTPATH/latest
+
+rm $TOOLLOC/dcraw
+
+mkdir $SCRIPTPATH/dcraw; cd dcraw
+wget --show-progress https://www.cybercom.net/~dcoffin/dcraw/dcraw.c
+gcc -o dcraw -O4 dcraw.c -lm -ljasper -ljpeg -llcms2
+
+mv dcraw $TOOLLOC/dcraw
+cd $SCRIPTPATH; rm -rf dcraw

tools/mkToolStack.sh

+#!/bin/bash
+
+SCRIPTPATH=$( cd $(dirname $0) ; pwd -P )
+ML_PATH="${SCRIPTPATH}/../src/magic-lantern"
+
+TOOLLOC=$SCRIPTPATH/latest
+
+rm -rf $SCRIPTPATH/old
+mv $TOOLLOC $SCRIPTPATH/old
+mkdir $TOOLLOC
+
+#Prepare source.
+cd $ML_PATH
+patch < ../patches/* #Apply patches.
+
+#Compile mlv_dump.
+cd $ML_PATH/modules/mlv_rec
+
+make clean; make mlv_dump -j
+
+cp mlv_dump $TOOLLOC
+
+make clean
+
+#Compile cr2hdr.
+cd $ML_PATH/modules/dual_iso
+
+make clean; make cr2hdr -j
+
+cp cr2hdr $TOOLLOC
+
+make clean
+
+#Compile raw2dng.
+cd $ML_PATH/modules/raw_rec
+
+make clean; make raw2dng -j
+
+cp raw2dng $TOOLLOC
+
+make clean
+
+#Make dcraw. Because why not.
+$SCRIPTPATH/mkDcraw.sh
+
+#Make Executable
+chmod -R +x $TOOLLOC/*
+
+#Cleanup
+cd $ML_PATH
+patch -R < ../patches/* #Remove patches.