From 751cc1b0c89d59bd347bd31715e553255b019a66 Mon Sep 17 00:00:00 2001 From: Revar Desmera Date: Sat, 1 Jun 2019 17:09:17 -0700 Subject: [PATCH] Added example script hashing for optimized image generation. --- scripts/docs_gen.py | 59 ++++++++++++++++++++++++++-------------- scripts/make_all_docs.sh | 22 +++++++++++---- 2 files changed, 56 insertions(+), 25 deletions(-) diff --git a/scripts/docs_gen.py b/scripts/docs_gen.py index 6f238d9..dc5c622 100755 --- a/scripts/docs_gen.py +++ b/scripts/docs_gen.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function @@ -7,6 +7,8 @@ import re import sys import math import random +import hashlib +import dbm.gnu import os.path import argparse import subprocess @@ -79,6 +81,7 @@ class ImageProcessing(object): self.commoncode = [] self.imgroot = "" self.keep_scripts = False + self.force = False def set_keep_scripts(self, x): self.keep_scripts = x @@ -89,12 +92,14 @@ class ImageProcessing(object): def set_commoncode(self, code): self.commoncode = code - def process_examples(self, imgroot): + def process_examples(self, imgroot, force=False): self.imgroot = imgroot - for libfile, imgfile, code, extype in self.examples: - self.gen_example_image(libfile, imgfile, code, extype) + self.force = force + with dbm.gnu.open("examples_hashes.gdbm", "c") as db: + for libfile, imgfile, code, extype in self.examples: + self.gen_example_image(db, libfile, imgfile, code, extype) - def gen_example_image(self, libfile, imgfile, code, extype): + def gen_example_image(self, db, libfile, imgfile, code, extype): OPENSCAD = "/Applications/OpenSCAD.app/Contents/MacOS/OpenSCAD" GIT = "/usr/local/bin/git" CONVERT = "/usr/local/bin/convert" @@ -103,7 +108,25 @@ class ImageProcessing(object): if extype == "NORENDER": return + print(" {}".format(imgfile)) + scriptfile = "tmp_{0}.scad".format(imgfile.replace(".", "_")) + targimgfile = self.imgroot + imgfile + newimgfile = self.imgroot + "_new_" + imgfile + + # Pull previous committed image from git, if it exists. + gitcmd = [GIT, "checkout", targimgfile] + p = subprocess.Popen(gitcmd, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) + err = p.stdout.read() + + m = hashlib.sha256() + m.update(extype.encode("utf8")) + for line in code: + m.update(line.encode("utf8")) + hash = m.digest() + key = "{0} - {1}".format(libfile, imgfile) + if key in db and db[key] == hash and not self.force: + return stdlibs = ["std.scad", "debug.scad"] script = "" @@ -128,8 +151,6 @@ class ImageProcessing(object): else: # Small imgsizes = ["480,360", "240x180"] - print(" {}".format(imgfile)) - tmpimgs = [] if "Spin" in extype: for ang in range(0,359,10): @@ -160,7 +181,7 @@ class ImageProcessing(object): p = subprocess.Popen(scadcmd, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) (stdoutdata, stderrdata) = p.communicate(None) res = p.returncode - if res != 0 or "ERROR:" in stderrdata or "WARNING:" in stderrdata: + if res != 0 or b"ERROR:" in stderrdata or b"WARNING:" in stderrdata: print("%s"%stderrdata) print("////////////////////////////////////////////////////") print("// {}: {} for {}".format(libfile, scriptfile, imgfile)) @@ -194,7 +215,7 @@ class ImageProcessing(object): p = subprocess.Popen(scadcmd, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) (stdoutdata, stderrdata) = p.communicate(None) res = p.returncode - if res != 0 or "ERROR:" in stderrdata or "WARNING:" in stderrdata: + if res != 0 or b"ERROR:" in stderrdata or b"WARNING:" in stderrdata: print("%s"%stderrdata) print("////////////////////////////////////////////////////") print("// {}: {} for {}".format(libfile, scriptfile, imgfile)) @@ -214,8 +235,6 @@ class ImageProcessing(object): if not self.keep_scripts: os.unlink(scriptfile) - targimgfile = self.imgroot + imgfile - newimgfile = self.imgroot + "_new_" + imgfile if len(tmpimgs) == 1: cnvcmd = [CONVERT, tmpimgfile, "-resize", imgsizes[1], newimgfile] @@ -243,11 +262,6 @@ class ImageProcessing(object): for tmpimg in tmpimgs: os.unlink(tmpimg) - # Pull previous committed image from git, if it exists. - gitcmd = [GIT, "checkout", targimgfile] - p = subprocess.Popen(gitcmd, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) - err = p.stdout.read() - # Time to compare image. if not os.path.isfile(targimgfile): print(" NEW IMAGE\n") @@ -260,13 +274,15 @@ class ImageProcessing(object): else: cmpcmd = [COMPARE, "-metric", "MAE", newimgfile, targimgfile, "null:"] p = subprocess.Popen(cmpcmd, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) - issame = p.stdout.read().strip() == "0 (0)" + resbin = p.stdout.read().strip() + issame = resbin == b'0 (0)' if issame: os.unlink(newimgfile) else: print(" UPDATED IMAGE\n") os.unlink(targimgfile) os.rename(newimgfile, targimgfile) + db[key] = hash imgprc = ImageProcessing() @@ -688,7 +704,7 @@ class LibFile(object): return out -def processFile(infile, outfile=None, gen_imgs=False, imgroot="", prefix=""): +def processFile(infile, outfile=None, gen_imgs=False, imgroot="", prefix="", force=False): if imgroot and not imgroot.endswith('/'): imgroot += "/" @@ -708,7 +724,7 @@ def processFile(infile, outfile=None, gen_imgs=False, imgroot="", prefix=""): print(line, file=f) if gen_imgs: - imgprc.process_examples(imgroot) + imgprc.process_examples(imgroot, force=force) if outfile: f.close() @@ -720,6 +736,8 @@ def main(): help="If given, don't delete the temporary image OpenSCAD scripts.") parser.add_argument('-c', '--comments-only', action="store_true", help='If given, only process lines that start with // comments.') + parser.add_argument('-f', '--force', action="store_true", + help='If given, force generation of images when the code is unchanged.') parser.add_argument('-i', '--images', action="store_true", help='If given, generate images for examples with OpenSCAD.') parser.add_argument('-I', '--imgroot', default="", @@ -735,7 +753,8 @@ def main(): outfile=args.outfile, gen_imgs=args.images, imgroot=args.imgroot, - prefix="// " if args.comments_only else "" + prefix="// " if args.comments_only else "", + force=args.force ) sys.exit(0) diff --git a/scripts/make_all_docs.sh b/scripts/make_all_docs.sh index 20bd7fc..24f4134 100755 --- a/scripts/make_all_docs.sh +++ b/scripts/make_all_docs.sh @@ -1,7 +1,18 @@ #!/bin/bash -if [[ $# > 0 ]]; then - PREVIEW_LIBS="$@" +FORCED="" +IMGGEN="" +FILES="" +for opt in "$@" ; do + case $opt in + -f ) FORCED=$opt ;; + -i ) IMGGEN=$opt ;; + * ) FILES="$FILES $opt" ;; + esac +done + +if [[ "$FILES" != "" ]]; then + PREVIEW_LIBS="$FILES" else PREVIEW_LIBS="common errors attachments math arrays vectors affine coords geometry triangulation quaternions hull constants edges transforms primitives shapes masks shapes2d paths beziers roundcorners walls metric_screws threading involute_gears sliders joiners linear_bearings nema_steppers wiring phillips_drive torx_drive polyhedra debug" fi @@ -18,10 +29,11 @@ rm -f tmpscad*.scad for lib in $PREVIEW_LIBS; do lib="$(basename $lib .scad)" mkdir -p images/$lib - rm -f images/$lib/*.png images/$lib/*.gif - # echo ../scripts/docs_gen.py ../$lib.scad -o $lib.scad.md -c -i -I images/$lib/ + if [ "$IMGGEN" = "-i" ]; then + rm -f images/$lib/*.png images/$lib/*.gif + fi echo "$lib.scad" - ../scripts/docs_gen.py ../$lib.scad -o $lib.scad.md -c -i -I images/$lib/ || exit 1 + ../scripts/docs_gen.py ../$lib.scad -o $lib.scad.md -c $IMGGEN $FORCED -I images/$lib/ || exit 1 open -a Typora $lib.scad.md done