BLD: use_build_cache.py - fix 2to3 caching and messages

Author: y-p

ci/before_install.sh

@@ -20,13 +20,8 @@ if  $PLEASE_TRAVIS_FASTER ; then
         unzip $ZIP_FLAGS /tmp/_"$CYTHON_HASH.zip" -d "$BUILD_CACHE_DIR";
         rm -f /tmp/_"$CYTHON_HASH.zip"
         # copy cythonized c files over
-        cp -R "$BUILD_CACHE_DIR"/pandas/* pandas/
-        # mkdir build/
-        # ls -l build
-        # mkdir build/temp.linux-x86-$TRAVIS_PYTHON_VERSION
-        # mkdir build/temp.linux-x86-$TRAVIS_PYTHON_VERSION/pandas/
-        # touch build/temp.linux-x86-$TRAVIS_PYTHON_VERSION/pandas/lib.o
-
+        cp -R "$BUILD_CACHE_DIR"/pandas/*.c pandas/
+        cp -R "$BUILD_CACHE_DIR"/pandas/src/*.c pandas/src/
     fi;
     echo "VENV_FILE_AVAILABLE=$VENV_FILE_AVAILABLE"
     if $VENV_FILE_AVAILABLE ; then

scripts/use_build_cache.py

@@ -78,18 +78,28 @@ class Foo(object):
         import shutil
         import multiprocessing
         pyver = "%d.%d" % (sys.version_info[:2])
-        files = ["pandas"]
+        fileq = ["pandas"]
         to_process = dict()
-        orig_hashes= dict((f.split("-")[0],f)  for f in os.listdir(BUILD_CACHE_DIR)
-                      if "-" in f and f.endswith(pyver))
-        post_hashes= dict((f.split("-")[1],f)  for f in os.listdir(BUILD_CACHE_DIR)
-                      if "-" in f and f.endswith(pyver))
 
-        while files:
-            f = files.pop()
+        # retrieve the hashes existing in the cache
+        orig_hashes=dict()
+        post_hashes=dict()
+        for path,dirs,files in os.walk(os.path.join(BUILD_CACHE_DIR,'pandas')):
+            for f in files:
+                s=f.split(".py-")[-1]
+                try:
+                    prev_h,post_h,ver = s.split('-')
+                    if ver == pyver:
+                        orig_hashes[prev_h] = os.path.join(path,f)
+                        post_hashes[post_h] = os.path.join(path,f)
+                except:
+                    pass
+
+        while fileq:
+            f = fileq.pop()
 
             if os.path.isdir(f):
-                files.extend([os.path.join(f,x) for x in os.listdir(f)])
+                fileq.extend([os.path.join(f,x) for x in os.listdir(f)])
             else:
                 if not f.endswith(".py"):
                     continue
@@ -98,39 +108,54 @@ class Foo(object):
                         h = sha1(open(f,"rb").read()).hexdigest()
                     except IOError:
                         to_process[h] = f
-                    if h in orig_hashes and not BC_FORCE_OVERWRITE:
-                        src = os.path.join(BUILD_CACHE_DIR,orig_hashes[h])
-                        if BC_DEBUG:
-                            print("2to3 cache hit %s,%s" % (f,h))
-                        shutil.copyfile(src,f)
-                    elif h not in post_hashes:
-
-                        # we're not in a dev dir with already processed files
-                        if BC_DEBUG:
-                            print("2to3 cache miss %s,%s" % (f,h))
-                            print("2to3 will process " + f)
-                        to_process[h] = f
+                    else:
+                        if h in orig_hashes and not BC_FORCE_OVERWRITE:
+                            src = orig_hashes[h]
+                            if BC_DEBUG:
+                                print("2to3 cache hit %s,%s" % (f,h))
+                            shutil.copyfile(src,f)
+                        elif h not in post_hashes:
+                            # we're not in a dev dir with already processed files
+                            if BC_DEBUG:
+                                print("2to3 cache miss (will process) %s,%s" % (f,h))
+                            to_process[h] = f
 
         avail_fixes = set(refactor.get_fixers_from_package("lib2to3.fixes"))
         avail_fixes.discard('lib2to3.fixes.fix_next')
         t=refactor.RefactoringTool(avail_fixes)
-        print("Starting 2to3 refactoring...")
-        for f in to_process.values():
-            if BC_DEBUG:
-                print("2to3 on %s" % f)
-            try:
-                t.refactor([f],True)
-                post_h = sha1(open(f, "rb").read()).hexdigest()
-                cached_fname = f + "-" + post_h + "-" + pyver
+        if to_process:
+            print("Starting 2to3 refactoring...")
+            for orig_h,f in to_process.items():
                 if BC_DEBUG:
-                    print("cache put %s,%s in %s" % (f, h, cached_fname))
-                shutil.copyfile(f, os.path.join(BUILD_CACHE_DIR, cached_fname))
+                    print("2to3 on %s" % f)
+                try:
+                    t.refactor([f],True)
+                    post_h = sha1(open(f, "rb").read()).hexdigest()
+                    cached_fname = f + '-' + orig_h  + '-' + post_h + '-' + pyver
+                    path = os.path.join(BUILD_CACHE_DIR, cached_fname)
+                    pathdir =os.path.dirname(path)
+                    if BC_DEBUG:
+                        print("cache put %s in %s" % (f, path))
+                    try:
+                        os.makedirs(pathdir)
+                    except OSError as exc:
+                        import errno
+                        if exc.errno == errno.EEXIST and os.path.isdir(pathdir):
+                            pass
+                        else:
+                            raise
 
-            except:
-                pass
-        print("2to3 done refactoring.")
+                    shutil.copyfile(f, path)
+
+                except Exception as e:
+                    print("While processing %s 2to3 raised: %s" % (f,str(e)))
+
+                    pass
+            print("2to3 done refactoring.")
 
 except Exception as e:
+    if not isinstance(e,ZeroDivisionError):
+        print( "Exception: " + str(e))
     BUILD_CACHE_DIR = None
 
 class CompilationCacheMixin(object):