preparing utility-script for gzip-littlefs

Software/platformio.ini

@@ -31,17 +31,18 @@ build_flags =
   -DADMIN_PASSWORD=${wifi_cred.admin_password}
   -DWIFI_AP_PASSWORD=${wifi_cred.wifi_ap_password}
   -DWIFI_AP_IP_GW=10,0,0,1
-  ;-DFEATURE_ENABLE_WIFI_CLIENT
+  -DFEATURE_ENABLE_WIFI_CLIENT
-  ;-DFEATURE_ENABLE_REMOTE_DEBUG
+  -DFEATURE_ENABLE_REMOTE_DEBUG
-  ;-DFEATURE_ENABLE_OLED
+  -DFEATURE_ENABLE_OLED
-  ;-DFEATURE_ENABLE_CAN
+  -DFEATURE_ENABLE_CAN
-  ;-DFEATURE_ENABLE_GPS
+  -DFEATURE_ENABLE_GPS
-  -DPCB_REV=2
+  -DPCB_REV=4
   -DNO_MODE_FLASH
 
 ;build_type = debug
 
 board_build.filesystem = littlefs
+extra_scripts = post:prepare_littlefs.py
 
 monitor_filters = esp8266_exception_decoder
 monitor_speed = 115200

Software/prepare_littlefs.py

@@ -0,0 +1,106 @@
+# SCRIPT TO GZIP CRITICAL FILES FOR ACCELERATED WEBSERVING
+# see also https://community.platformio.org/t/question-esp32-compress-files-in-data-to-gzip-before-upload-possible-to-spiffs/6274/10
+
+
+import glob
+import shutil
+import gzip
+import os
+Import("env")
+Import("projenv")
+
+
+def gzip_file(src_path, dst_path):
+
+    with open(src_path, 'rb') as src, gzip.open(dst_path, 'wb') as dst:
+        for chunk in iter(lambda: src.read(4096), b""):
+            dst.write(chunk)
+
+
+def getListOfFiles(dirName):
+    # create a list of file and sub directories
+    # names in the given directory
+    listOfFile = os.listdir(dirName)
+    allFiles = list()
+    # Iterate over all the entries
+    for entry in listOfFile:
+        # Create full path
+        fullPath = os.path.join(dirName, entry)
+        # If entry is a directory then get the list of files in this directory
+        if os.path.isdir(fullPath):
+            allFiles = allFiles + getListOfFiles(fullPath)
+        else:
+            allFiles.append(fullPath)
+
+    return allFiles
+
+def remove_prefix(text, prefix):
+    if text.startswith(prefix):
+        return text[len(prefix):]
+    return text  # or whatever
+
+# Compress files from 'data_src/' to 'data/'
+
+
+def gzip_webfiles(source, target, env):
+    # Filetypes to compress
+    filetypes_to_gzip = ['.css', '.png', '.js', '.ico']
+    print('\nGZIP: Starting gzip-Process for LittleFS-Image...\n')
+    data_src_dir_path = os.path.join(env.get('PROJECT_DIR'), 'data_src')
+    data_dir_path = env.get('PROJECT_DATA_DIR')
+    # check if data and datasrc exist. If the first exists and not the second, it renames it
+    if(os.path.exists(data_dir_path) and not os.path.exists(data_src_dir_path)):
+        print('GZIP: Directory "'+data_dir_path +
+              '" exists, "'+data_src_dir_path+'" is not found.')
+        print('GZIP: Renaming "' + data_dir_path +
+              '" to "' + data_src_dir_path + '"')
+        os.rename(data_dir_path, data_src_dir_path)
+    # Delete the 'data' directory
+    if(os.path.exists(data_dir_path)):
+        print('GZIP: Deleting the "data" directory ' + data_dir_path)
+        shutil.rmtree(data_dir_path)
+    # Recreate empty 'data' directory
+    print('GZIP: Re-creating an empty data directory ' + data_dir_path)
+    os.mkdir(data_dir_path)
+    # Determine the files to compress
+
+    files_to_copy = []
+    files_to_gzip = []
+
+    all_data_src = getListOfFiles(data_src_dir_path)
+    for file in all_data_src:
+        file_name, file_extension = os.path.splitext(file)
+        print(file_name + " has filetype " + file_extension)
+        if file_extension in filetypes_to_gzip:
+            files_to_gzip.append(file)
+        else:
+            filename_subdir = remove_prefix(file, data_src_dir_path)
+            files_to_copy.append(filename_subdir)
+
+    for file in files_to_copy:
+        print('GZIP: Copying file from: ' + data_src_dir_path + file + ' to: ' + data_dir_path + file)
+        os.makedirs(os.path.dirname(data_dir_path + file), exist_ok=True)
+        shutil.copy(data_src_dir_path + file, data_dir_path + file)
+    # Compress and move files
+    
+    was_error = False
+    try:
+        for source_file_path in files_to_gzip:
+            print('GZIP: compressing... ' + source_file_path)
+            filename_subdir = remove_prefix(source_file_path, data_src_dir_path)
+            target_file_path = data_dir_path + filename_subdir
+            os.makedirs(os.path.dirname(target_file_path), exist_ok=True)
+            print('GZIP: Compressed... ' + target_file_path)
+            gzip_file(source_file_path, target_file_path + ".gz")
+    except IOError as e:
+        was_error = True
+        print('GZIP: Failed to compress file: ' + source_file_path)
+        # print( 'GZIP: EXCEPTION... {}'.format( e ) )
+    if was_error:
+        print('GZIP: Failure/Incomplete.\n')
+    else:
+        print('GZIP: Compressed correctly.\n')
+ 
+
+# IMPORTANT, this needs to be added to call the routine
+env.AddPreAction('$BUILD_DIR/littlefs.bin', gzip_webfiles)