# SCRIPT TO GZIP CRITICAL FILES FOR ACCELERATED WEBSERVING # see also https://community.platformio.org/t/question-esp32-compress-files-in-data-to-gzip-before-upload-possible-to-spiffs/6274/10 import glob import shutil import gzip import os Import("env") Import("projenv") def gzip_file(src_path, dst_path): with open(src_path, 'rb') as src, gzip.open(dst_path, 'wb') as dst: for chunk in iter(lambda: src.read(4096), b""): dst.write(chunk) def getListOfFiles(dirName): # create a list of file and sub directories # names in the given directory listOfFile = os.listdir(dirName) allFiles = list() # Iterate over all the entries for entry in listOfFile: # Create full path fullPath = os.path.join(dirName, entry) # If entry is a directory then get the list of files in this directory if os.path.isdir(fullPath): allFiles = allFiles + getListOfFiles(fullPath) else: allFiles.append(fullPath) return allFiles def remove_prefix(text, prefix): if text.startswith(prefix): return text[len(prefix):] return text # or whatever # Compress files from 'data_src/' to 'data/' def gzip_webfiles(source, target, env): # Filetypes to compress filetypes_to_gzip = ['.css', '.png', '.js', '.ico', '.woff2', '.json'] print('\nGZIP: Starting gzip-Process for LittleFS-Image...\n') data_src_dir_path = os.path.join(env.get('PROJECT_DIR'), 'data_src') data_dir_path = env.get('PROJECT_DATA_DIR') # check if data and datasrc exist. If the first exists and not the second, it renames it if(os.path.exists(data_dir_path) and not os.path.exists(data_src_dir_path)): print('GZIP: Directory "'+data_dir_path + '" exists, "'+data_src_dir_path+'" is not found.') print('GZIP: Renaming "' + data_dir_path + '" to "' + data_src_dir_path + '"') os.rename(data_dir_path, data_src_dir_path) # Delete the 'data' directory if(os.path.exists(data_dir_path)): print('GZIP: Deleting the "data" directory ' + data_dir_path) shutil.rmtree(data_dir_path) # Recreate empty 'data' directory print('GZIP: Re-creating an empty data directory ' + data_dir_path) os.mkdir(data_dir_path) # Determine the files to compress files_to_copy = [] files_to_gzip = [] all_data_src = getListOfFiles(data_src_dir_path) for file in all_data_src: file_name, file_extension = os.path.splitext(file) print(file_name + " has filetype " + file_extension) if file_extension in filetypes_to_gzip: files_to_gzip.append(file) else: filename_subdir = remove_prefix(file, data_src_dir_path) files_to_copy.append(filename_subdir) for file in files_to_copy: print('GZIP: Copying file from: ' + data_src_dir_path + file + ' to: ' + data_dir_path + file) os.makedirs(os.path.dirname(data_dir_path + file), exist_ok=True) shutil.copy(data_src_dir_path + file, data_dir_path + file) # Compress and move files was_error = False try: for source_file_path in files_to_gzip: print('GZIP: compressing... ' + source_file_path) filename_subdir = remove_prefix(source_file_path, data_src_dir_path) target_file_path = data_dir_path + filename_subdir os.makedirs(os.path.dirname(target_file_path), exist_ok=True) print('GZIP: Compressed... ' + target_file_path) gzip_file(source_file_path, target_file_path + ".gz") except IOError as e: was_error = True print('GZIP: Failed to compress file: ' + source_file_path) # print( 'GZIP: EXCEPTION... {}'.format( e ) ) if was_error: print('GZIP: Failure/Incomplete.\n') else: print('GZIP: Compressed correctly.\n') return def gzip_binffiles(source, target, env): littlefsbin = target[0].get_abspath() targetbin = os.path.join(os.path.dirname(littlefsbin), 'filesystem.fs') shutil.copyfile(littlefsbin, targetbin) gzip_file(targetbin, os.path.join(str(targetbin) + '.gz')) os.remove(targetbin) return # IMPORTANT, this needs to be added to call the routine env.AddPreAction('$BUILD_DIR/littlefs.bin', gzip_webfiles) env.AddPostAction('$BUILD_DIR/littlefs.bin', gzip_binffiles)