diff --git a/scripts/windows_builder/build_26-32.bat b/scripts/windows_builder/build_26-32.bat new file mode 100644 index 0000000000000..00cf016ff3bad --- /dev/null +++ b/scripts/windows_builder/build_26-32.bat @@ -0,0 +1,21 @@ +@echo off +echo "starting 26-32" + + +title 26-32 build +echo "building" +cd "c:\users\Jeff Reback\documents\github\pandas" +C:\python26-32\python.exe setup.py build > build.26-32.log 2>&1 + +echo "installing" +C:\python26-32\python.exe setup.py bdist --formats=wininst > install.26-32.log 2>&1 + +echo "testing" +C:\python26-32\scripts\nosetests -A "not slow" build\lib.win32-2.6\pandas > test.26-32.log 2>&1 + +echo "versions" +cd build\lib.win32-2.6 +C:\python26-32\python.exe ../../ci/print_versions.py > ../../versions.26-32.log 2>&1 + + +exit diff --git a/scripts/windows_builder/build_26-64.bat b/scripts/windows_builder/build_26-64.bat new file mode 100644 index 0000000000000..55abf37c6c37a --- /dev/null +++ b/scripts/windows_builder/build_26-64.bat @@ -0,0 +1,25 @@ +@echo off +echo "starting 26-64" + +setlocal EnableDelayedExpansion +set MSSdk=1 +CALL "C:\Program Files\Microsoft SDKs\Windows\v7.0\Bin\SetEnv.cmd" /x64 /release +set DISTUTILS_USE_SDK=1 + +title 26-64 build +echo "building" +cd "c:\users\Jeff Reback\documents\github\pandas" +C:\python26-64\python.exe setup.py build > build.26-64.log 2>&1 + +echo "installing" +C:\python26-64\python.exe setup.py bdist --formats=wininst > install.26-64.log 2>&1 + +echo "testing" +C:\python26-64\scripts\nosetests -A "not slow" build\lib.win-amd64-2.6\pandas > test.26-64.log 2>&1 + +echo "versions" +cd build\lib.win-amd64-2.6 +C:\python26-64\python.exe ../../ci/print_versions.py > ../../versions.26-64.log 2>&1 + + +exit diff --git a/scripts/windows_builder/build_27-32.bat b/scripts/windows_builder/build_27-32.bat new file mode 100644 index 0000000000000..37eb4d436d567 --- /dev/null +++ b/scripts/windows_builder/build_27-32.bat @@ -0,0 +1,25 @@ +@echo off +echo "starting 27-32" + +setlocal EnableDelayedExpansion +set MSSdk=1 +CALL "C:\Program Files\Microsoft SDKs\Windows\v7.0\Bin\SetEnv.cmd" /x86 /release +set DISTUTILS_USE_SDK=1 + +title 27-32 build +echo "building" +cd "c:\users\Jeff Reback\documents\github\pandas" +C:\python27-32\python.exe setup.py build > build.27-32.log 2>&1 + +title "installing" +C:\python27-32\python.exe setup.py bdist --formats=wininst > install.27-32.log 2>&1 + +echo "testing" +C:\python27-32\scripts\nosetests -A "not slow" build\lib.win32-2.7\pandas > test.27-32.log 2>&1 + +echo "versions" +cd build\lib.win32-2.7 +C:\python27-32\python.exe ../../ci/print_versions.py > ../../versions.27-32.log 2>&1 + +exit + diff --git a/scripts/windows_builder/build_27-64.bat b/scripts/windows_builder/build_27-64.bat new file mode 100644 index 0000000000000..e76e25d0ef39c --- /dev/null +++ b/scripts/windows_builder/build_27-64.bat @@ -0,0 +1,25 @@ +@echo off +echo "starting 27-64" + +setlocal EnableDelayedExpansion +set MSSdk=1 +CALL "C:\Program Files\Microsoft SDKs\Windows\v7.0\Bin\SetEnv.cmd" /x64 /release +set DISTUTILS_USE_SDK=1 + +title 27-64 build +echo "building" +cd "c:\users\Jeff Reback\documents\github\pandas" +C:\python27-64\python.exe setup.py build > build.27-64.log 2>&1 + +echo "installing" +C:\python27-64\python.exe setup.py bdist --formats=wininst > install.27-64.log 2>&1 + +echo "testing" +C:\python27-64\scripts\nosetests -A "not slow" build\lib.win-amd64-2.7\pandas > test.27-64.log 2>&1 + +echo "versions" +cd build\lib.win-amd64-2.7 +C:\python27-64\python.exe ../../ci/print_versions.py > ../../versions.27-64.log 2>&1 + +exit + diff --git a/scripts/windows_builder/build_33-32.bat b/scripts/windows_builder/build_33-32.bat new file mode 100644 index 0000000000000..cf629bc71f34e --- /dev/null +++ b/scripts/windows_builder/build_33-32.bat @@ -0,0 +1,27 @@ +@echo off +echo "starting 33-32" + +setlocal EnableDelayedExpansion +set MSSdk=1 +CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x86 /release +set DISTUTILS_USE_SDK=1 + +title 33-32 build +echo "building" +cd "c:\users\Jeff Reback\documents\github\pandas" +C:\python33-32\python.exe setup.py build > build.33-32.log 2>&1 + +echo "installing" +C:\python33-32\python.exe setup.py bdist --formats=wininst > install.33-32.log 2>&1 + +echo "testing" +C:\python33-32\scripts\nosetests -A "not slow" build\lib.win32-3.3\pandas > test.33-32.log 2>&1 + +echo "versions" +cd build\lib.win32-3.3 +C:\python33-32\python.exe ../../ci/print_versions.py > ../../versions.33-32.log 2>&1 + +exit + + + diff --git a/scripts/windows_builder/build_33-64.bat b/scripts/windows_builder/build_33-64.bat new file mode 100644 index 0000000000000..8f037941868f9 --- /dev/null +++ b/scripts/windows_builder/build_33-64.bat @@ -0,0 +1,27 @@ +@echo off +echo "starting 33-64" + +setlocal EnableDelayedExpansion +set MSSdk=1 +CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 /release +set DISTUTILS_USE_SDK=1 + +title 33-64 build +echo "building" +cd "c:\users\Jeff Reback\documents\github\pandas" +C:\python33-64\python.exe setup.py build > build.33-64.log 2>&1 + +echo "installing" +C:\python33-64\python.exe setup.py bdist --formats=wininst > install.33-64.log 2>&1 + +echo "testing" +C:\python33-64\scripts\nosetests -A "not slow" build\lib.win-amd64-3.3\pandas > test.33-64.log 2>&1 + +echo "versions" +cd build\lib.win-amd64-3.3 +C:\python33-64\python.exe ../../ci/print_versions.py > ../../versions.33-64.log 2>&1 + +exit + + + diff --git a/scripts/windows_builder/build_34-32.bat b/scripts/windows_builder/build_34-32.bat new file mode 100644 index 0000000000000..8e060e000bc8f --- /dev/null +++ b/scripts/windows_builder/build_34-32.bat @@ -0,0 +1,27 @@ +@echo off +echo "starting 34-32" + +setlocal EnableDelayedExpansion +set MSSdk=1 +CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x86 /release +set DISTUTILS_USE_SDK=1 + +title 34-32 build +echo "building" +cd "c:\users\Jeff Reback\documents\github\pandas" +C:\python34-32\python.exe setup.py build > build.34-32.log 2>&1 + +echo "installing" +C:\python34-32\python.exe setup.py bdist --formats=wininst > install.34-32.log 2>&1 + +echo "testing" +C:\python34-32\scripts\nosetests -A "not slow" build\lib.win32-3.4\pandas > test.34-32.log 2>&1 + +echo "versions" +cd build\lib.win32-3.4 +C:\python34-32\python.exe ../../ci/print_versions.py > ../../versions.34-32.log 2>&1 + +exit + + + diff --git a/scripts/windows_builder/build_34-64.bat b/scripts/windows_builder/build_34-64.bat new file mode 100644 index 0000000000000..3a8512b730346 --- /dev/null +++ b/scripts/windows_builder/build_34-64.bat @@ -0,0 +1,27 @@ +@echo off +echo "starting 34-64" + +setlocal EnableDelayedExpansion +set MSSdk=1 +CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 /release +set DISTUTILS_USE_SDK=1 + +title 34-64 build +echo "building" +cd "c:\users\Jeff Reback\documents\github\pandas" +C:\python34-64\python.exe setup.py build > build.34-64.log 2>&1 + +echo "installing" +C:\python34-64\python.exe setup.py bdist --formats=wininst > install.34-64.log 2>&1 + +echo "testing" +C:\python34-64\scripts\nosetests -A "not slow" build\lib.win-amd64-3.4\pandas > test.34-64.log 2>&1 + +echo "versions" +cd build\lib.win-amd64-3.4 +C:\python34-64\python.exe ../../ci/print_versions.py > ../../versions.34-64.log 2>&1 + +exit + + + diff --git a/scripts/windows_builder/check_and_build.bat b/scripts/windows_builder/check_and_build.bat new file mode 100644 index 0000000000000..32be1bde1f7f3 --- /dev/null +++ b/scripts/windows_builder/check_and_build.bat @@ -0,0 +1,2 @@ +set PYTHONPATH=c:/python27-64/lib +c:/python27-64/python.exe c:/Builds/check_and_build.py %1 %2 %3 %4 %4 %6 %7 %8 %9 diff --git a/scripts/windows_builder/check_and_build.py b/scripts/windows_builder/check_and_build.py new file mode 100644 index 0000000000000..81669972b991d --- /dev/null +++ b/scripts/windows_builder/check_and_build.py @@ -0,0 +1,194 @@ +import datetime +import git +import logging +import os, re, time +import subprocess +import argparse +import pysftp + +# parse the args +parser = argparse.ArgumentParser(description='build, test, and install updated versions of master pandas') +parser.add_argument('-b', '--build', + help='run just this build', + dest='build') +parser.add_argument('-u', '--update', + help='get a git update', + dest='update', + action='store_true', + default=False) +parser.add_argument('-t', '--test', + help='run the tests', + dest='test', + action='store_true', + default=False) +parser.add_argument('-c', '--compare', + help='show the last tests compare', + dest='compare', + action='store_true', + default=False) +parser.add_argument('-v', '--version', + help='show the last versions', + dest='version', + action='store_true', + default=False) +parser.add_argument('-i', '--install', + help='run the install', + dest='install', + action='store_true', + default=False) +parser.add_argument('--dry', + help='dry run', + dest='dry', + action='store_true', + default=False) + +args = parser.parse_args() +dry_run = args.dry + +builds = ['26-32','26-64','27-32','27-64','33-32','33-64','34-32','34-64'] +base_dir = "C:\Users\Jeff Reback\Documents\GitHub\pandas" +remote_host='pandas.pydata.org' +username='pandas' +password=############ + +# drop python from our environment to avoid +# passing this onto sub-processes +env = os.environ +del env['PYTHONPATH'] + +# the stdout logger +fmt = '%(asctime)s: %(message)s' +logger = logging.getLogger('check_and_build') +logger.setLevel(logging.DEBUG) +stream_handler = logging.StreamHandler() +stream_handler.setFormatter(logging.Formatter(fmt)) +logger.addHandler(stream_handler) + +def run_all(test=False,compare=False,install=False,version=False,build=None): + # run everything + + for b in builds: + if build is not None and build != b: + continue + if test: + do_rebuild(b) + if compare or test: + try: + do_compare(b) + except (Exception) as e: + logger.info("ERROR COMPARE {0} : {1}".format(b,e)) + if version: + try: + do_version(b) + except (Exception) as e: + logger.info("ERROR VERSION {0} : {1}".format(b,e)) + + if install: + run_install() + +def do_rebuild(build): + # trigger the rebuild + + cmd = "c:/Builds/build_{0}.bat".format(build) + logger.info("rebuild : {0}".format(cmd)) + p = subprocess.Popen("start /wait /min {0}".format(cmd),env=env,shell=True,close_fds=True) + ret = p.wait() + +def do_compare(build): + # print the test outputs + + f = os.path.join(base_dir,"test.{0}.log".format(build)) + with open(f,'r') as fh: + for l in fh: + l = l.rstrip() + if l.startswith('ERROR:'): + logger.info("{0} : {1}".format(build,l)) + if l.startswith('Ran') or l.startswith('OK') or l.startswith('FAIL'): + logger.info("{0} : {1}".format(build,l)) + +def do_version(build): + # print the version strings + + f = os.path.join(base_dir,"versions.{0}.log".format(build)) + with open(f,'r') as fh: + for l in fh: + l = l.rstrip() + logger.info("{0} : {1}".format(build,l)) + +def do_update(is_verbose=True): + # update git; return True if the commit has changed + + repo = git.Repo(base_dir) + master = repo.heads.master + origin = repo.remotes.origin + start_commit = master.commit + + if is_verbose: + logger.info("current commit : {0}".format(start_commit)) + + try: + origin.update() + except (Exception) as e: + logger.info("update exception : {0}".format(e)) + try: + origin.pull() + except (Exception) as e: + logger.info("pull exception : {0}".format(e)) + + result = start_commit != master.commit + if result: + if is_verbose: + logger.info("commits changed : {0} -> {1}".format(start_commit,master.commit)) + return result + +def run_install(): + # send the installation binaries + + repo = git.Repo(base_dir) + master = repo.heads.master + commit = master.commit + short_hash = str(commit)[:7] + + logger.info("sending files : {0}".format(commit)) + d = os.path.join(base_dir,"dist") + files = [ f for f in os.listdir(d) if re.search(short_hash,f) ] + srv = pysftp.Connection(host=remote_host,username=username,password=password) + srv.chdir("www/pandas-build/dev") + + # get current files + remote_files = set(srv.listdir(path='.')) + + for f in files: + if f not in remote_files: + logger.info("sending: {0}".format(f)) + local = os.path.join(d,f) + srv.put(localpath=local) + + srv.close() + logger.info("sending files: done") + +# just perform the action +if args.update or args.test or args.compare or args.install or args.version: + if args.update: + do_update() + run_all(test=args.test,compare=args.compare,install=args.install,version=args.version,build=args.build) + exit(0) + +# file logging +file_handler = logging.FileHandler("C:\Builds\logs\check_and_build.log") +file_handler.setFormatter(logging.Formatter(fmt)) +logger.addHandler(file_handler) + +logger.info("start") + +# main loop +while(True): + + if do_update(): + run_all(test=True,install=False) + + time.sleep(60*60) + +logger.info("exit") +file_handler.close() + diff --git a/scripts/windows_builder/readme.txt b/scripts/windows_builder/readme.txt new file mode 100644 index 0000000000000..85c011e515b74 --- /dev/null +++ b/scripts/windows_builder/readme.txt @@ -0,0 +1,17 @@ +This is a collection of windows batch scripts (and a python script) +to rebuild the binaries, test, and upload the binaries for public distribution +upon a commit on github. + +Obviously requires that these be setup on windows +Requires an install of Windows SDK 3.5 and 4.0 +Full python installs for each version with the deps + +Currently supporting + +26-32,26-64,27-32,27-64,33-32,33-64,34-32,34-64 + +Note that 33 and 34 use the 4.0 SDK, while the other suse 3.5 SDK + +I installed these scripts in C:\Builds + +Installed libaries in C:\Installs