Refactor export subsystem

Makes several broad changes:
 - removes dead code that dealt with the online build system
 - replaces export function with a much simpler one that:
   - does not copy any sources
   - the zip file hits the disk
   - the mbed_config.h hits the disk
   - the project files hit the disk
   - nothing else hits the disk
 - exporters use Resource object scanned with a toolchain
 - progen exporters don't optionally build a project instead they have a
   build function that may be called afterwards
 - much of the code passes pylint (have a score of 9 or above):
   - project.py
   - project_api.py
   - export/__init__.py
   - export/exporters.py
   - test/export/build_test.py

Author: sarahmarshy

tools/build_api.py

@@ -252,7 +252,7 @@ def prepare_toolchain(src_paths, target, toolchain_name,
 
     return toolchain
 
-def scan_resources(src_paths, toolchain, dependencies_paths=None, inc_dirs=None):
+def scan_resources(src_paths, toolchain, dependencies_paths=None, inc_dirs=None, base_path=None):
     """ Scan resources using initialized toolcain
     src_paths: the paths to source directories
     toolchain: valid toolchain object
@@ -261,9 +261,9 @@ def scan_resources(src_paths, toolchain, dependencies_paths=None, inc_dirs=None)
     """
 
     # Scan src_path
-    resources = toolchain.scan_resources(src_paths[0])
+    resources = toolchain.scan_resources(src_paths[0], base_path=base_path)
     for path in src_paths[1:]:
-        resources.add(toolchain.scan_resources(path))
+        resources.add(toolchain.scan_resources(path, base_path=base_path))
 
     # Scan dependency paths for include dirs
     if dependencies_paths is not None:
@@ -495,7 +495,7 @@ def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean
     """
     lib = Library(lib_id)
     if not lib.is_supported(target, toolchain_name):
-        print 'Library "%s" is not yet supported on target %s with toolchain %s' % (lib_id, target.name, toolchain)
+        print 'Library "%s" is not yet supported on target %s with toolchain %s' % (lib_id, target.name, toolchain_name)
         return False
 
     # We need to combine macros from parameter list with macros from library definition

tools/export/__init__.py

@@ -1,27 +1,28 @@
+"""The generic interface for all exporters.
 """
-mbed SDK
-Copyright (c) 2011-2013 ARM Limited
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
+# mbed SDK
+# Copyright (c) 2011-2013 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 import os, tempfile
 from os.path import join, exists, basename
 from shutil import copytree, rmtree, copy
 import yaml
 
-from tools.utils import mkdir
-from tools.export import uvision4, uvision5, codered, gccarm, ds5_5, iar, emblocks, coide, kds, zip, simplicityv3, atmelstudio, sw4stm32, e2studio
-from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException, FailedBuildException
+from tools.export import uvision4, uvision5, codered, gccarm, ds5_5, iar
+from tools.export import emblocks, coide, kds, simplicityv3, atmelstudio
+from tools.export import sw4stm32, e2studio
+from tools.export.exporters import OldLibrariesException, FailedBuildException
 from tools.targets import TARGET_NAMES, EXPORT_MAP, TARGET_MAP
 
 from project_generator_definitions.definitions import ProGenDef
@@ -52,162 +53,25 @@
 To export this project please <a href='http://mbed.org/compiler/?import=http://mbed.org/users/mbed_official/code/mbed-export/k&mode=lib' target='_blank'>import the export version of the mbed library</a>.
 """
 
-def online_build_url_resolver(url):
-    # TODO: Retrieve the path and name of an online library build URL
-    return {'path':'', 'name':''}
-
-
-def export(project_path, project_name, ide, target, destination='/tmp/',
-           tempdir=None, pgen_build = False, clean=True, extra_symbols=None, make_zip=True, sources_relative=False,
-           build_url_resolver=online_build_url_resolver, progen_build=False):
-    # Convention: we are using capitals for toolchain and target names
-    if target is not None:
-        target = target.upper()
-
-    if tempdir is None:
-        tempdir = tempfile.mkdtemp()
-
-    use_progen = False
-    supported = True
-    report = {'success': False, 'errormsg':'', 'skip': False}
-
-    if ide is None or ide == "zip":
-        # Simple ZIP exporter
-        try:
-            ide = "zip"
-            exporter = zip.ZIP(target, tempdir, project_name, build_url_resolver, extra_symbols=extra_symbols)
-            exporter.scan_and_copy_resources(project_path, tempdir, sources_relative)
-            exporter.generate()
-            report['success'] = True
-        except OldLibrariesException, e:
-            report['errormsg'] = ERROR_MESSAGE_NOT_EXPORT_LIBS
-    else:
-        if ide not in EXPORTERS:
-            report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide)
-            report['skip'] = True
-        else:
-            Exporter = EXPORTERS[ide]
-            target = EXPORT_MAP.get(target, target)
-            try:
-                if Exporter.PROGEN_ACTIVE:
-                    use_progen = True
-            except AttributeError:
-                pass
-
-            if target not in Exporter.TARGETS or Exporter.TOOLCHAIN not in TARGET_MAP[target].supported_toolchains:
-                supported = False
-
-            if use_progen:
-                if not ProGenDef(ide).is_supported(TARGET_MAP[target].progen['target']):
-                    supported = False
-
-            if supported:
-                # target checked, export
-                try:
-                    exporter = Exporter(target, tempdir, project_name, build_url_resolver, extra_symbols=extra_symbols, sources_relative=sources_relative)
-                    exporter.scan_and_copy_resources(project_path, tempdir, sources_relative)
-                    if progen_build:
-                        #try to build with pgen ide builders
-                        try:
-                            exporter.generate(progen_build=True)
-                            report['success'] = True
-                        except FailedBuildException, f:
-                            report['errormsg'] = "Build Failed"
-                    else:
-                        exporter.generate()
-                        report['success'] = True
-                except OldLibrariesException, e:
-                    report['errormsg'] = ERROR_MESSAGE_NOT_EXPORT_LIBS
-
-            else:
-                report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide)
-                report['skip'] = True
+def mcu_ide_matrix(verbose_html=False):
+    """Shows target map using prettytable
 
-    zip_path = None
-    if report['success']:
-        # readme.txt to contain more exported data
-        exporter_yaml = { 
-            'project_generator': {
-                'active' : False,
-            }
-        }
-        if use_progen:
-            try:
-                import pkg_resources
-                version = pkg_resources.get_distribution('project_generator').version
-                exporter_yaml['project_generator']['version'] = version
-                exporter_yaml['project_generator']['active'] =  True;
-                exporter_yaml['project_generator_definitions'] = {}
-                version = pkg_resources.get_distribution('project_generator_definitions').version
-                exporter_yaml['project_generator_definitions']['version'] = version
-            except ImportError:
-                pass
-        with open(os.path.join(tempdir, 'exporter.yaml'), 'w') as outfile:
-            yaml.dump(exporter_yaml, outfile, default_flow_style=False)
-        # add readme file to every offline export.
-        open(os.path.join(tempdir, 'GettingStarted.htm'),'w').write('<meta http-equiv="refresh" content="0; url=http://mbed.org/handbook/Getting-Started-mbed-Exporters#%s"/>'% (ide))
-        # copy .hgignore file to exported direcotry as well.
-        if exists(os.path.join(exporter.TEMPLATE_DIR,'.hgignore')):
-            copy(os.path.join(exporter.TEMPLATE_DIR,'.hgignore'), tempdir)
-        if make_zip:
-            zip_path = zip_working_directory_and_clean_up(tempdir, destination, project_name, clean)
-        else:
-            zip_path = destination
-
-    return zip_path, report
-
-
-###############################################################################
-# Generate project folders following the online conventions
-###############################################################################
-def copy_tree(src, dst, clean=True):
-    if exists(dst):
-        if clean:
-            rmtree(dst)
-        else:
-            return
-
-    copytree(src, dst)
-
-
-def setup_user_prj(user_dir, prj_path, lib_paths=None):
-    """
-    Setup a project with the same directory structure of the mbed online IDE
+    Keyword argumets:
+    verbose_html - print the matrix in html format
     """
-    mkdir(user_dir)
-
-    # Project Path
-    copy_tree(prj_path, join(user_dir, "src"))
-
-    # Project Libraries
-    user_lib = join(user_dir, "lib")
-    mkdir(user_lib)
-
-    if lib_paths is not None:
-        for lib_path in lib_paths:
-            copy_tree(lib_path, join(user_lib, basename(lib_path)))
-
-def mcu_ide_matrix(verbose_html=False, platform_filter=None):
-    """  Shows target map using prettytable """
-    supported_ides = []
-    for key in EXPORTERS.iterkeys():
-        supported_ides.append(key)
-    supported_ides.sort()
-    from prettytable import PrettyTable, ALL # Only use it in this function so building works without extra modules
+    supported_ides = sorted(EXPORTERS.keys())
+    # Only use it in this function so building works without extra modules
+    from prettytable import PrettyTable, ALL
 
     # All tests status table print
-    columns = ["Platform"] + supported_ides
-    pt = PrettyTable(columns)
+    table_printer = PrettyTable(["Platform"] + supported_ides)
     # Align table
-    for col in columns:
-        pt.align[col] = "c"
-    pt.align["Platform"] = "l"
+    for col in supported_ides:
+        table_printer.align[col] = "c"
+    table_printer.align["Platform"] = "l"
 
     perm_counter = 0
-    target_counter = 0
     for target in sorted(TARGET_NAMES):
-        target_counter += 1
-
         row = [target]  # First column is platform name
         for ide in supported_ides:
             text = "-"
@@ -218,20 +82,24 @@ def mcu_ide_matrix(verbose_html=False, platform_filter=None):
                     text = "x"
                 perm_counter += 1
             row.append(text)
-        pt.add_row(row)
+        table_printer.add_row(row)
 
-    pt.border = True
-    pt.vrules = ALL
-    pt.hrules = ALL
-    # creates a html page suitable for a browser
-    # result = pt.get_html_string(format=True) if verbose_html else pt.get_string()
+    table_printer.border = True
+    table_printer.vrules = ALL
+    table_printer.hrules = ALL
     # creates a html page in a shorter format suitable for readme.md
-    result = pt.get_html_string() if verbose_html else pt.get_string()
+    if verbose_html:
+        result = table_printer.get_html_string()
+    else:
+        result = table_printer.get_string()
     result += "\n"
     result += "Total IDEs: %d\n"% (len(supported_ides))
-    if verbose_html: result += "<br>"
-    result += "Total platforms: %d\n"% (target_counter)
-    if verbose_html: result += "<br>"
+    if verbose_html:
+        result += "<br>"
+    result += "Total platforms: %d\n"% (len(TARGET_NAMES))
+    if verbose_html:
+        result += "<br>"
     result += "Total permutations: %d"% (perm_counter)
-    if verbose_html: result = result.replace("&amp;", "&")
+    if verbose_html:
+        result = result.replace("&amp;", "&")
     return result

tools/export/atmelstudio.py

@@ -61,7 +61,7 @@ def generate(self):
 
         ctx = {
             'target': self.target,
-            'name': self.program_name,
+            'name': self.project_name,
             'source_files': source_files,
             'source_folders': source_folders,
             'object_files': self.resources.objects,
@@ -73,7 +73,7 @@ def generate(self):
             'solution_uuid': solution_uuid.upper(),
             'project_uuid': project_uuid.upper()
         }
-        ctx.update(self.progen_flags)
+        ctx.update(self.flags)
         target = self.target.lower()
-        self.gen_file('atmelstudio6_2.atsln.tmpl', ctx, '%s.atsln' % self.program_name)
-        self.gen_file('atmelstudio6_2.cppproj.tmpl', ctx, '%s.cppproj' % self.program_name)
+        self.gen_file('atmelstudio6_2.atsln.tmpl', ctx, '%s.atsln' % self.project_name)
+        self.gen_file('atmelstudio6_2.cppproj.tmpl', ctx, '%s.cppproj' % self.project_name)

tools/export/codered.py

@@ -48,13 +48,13 @@ def generate(self):
             libraries.append(l[3:])
 
         ctx = {
-            'name': self.program_name,
+            'name': self.project_name,
             'include_paths': self.resources.inc_dirs,
             'linker_script': self.resources.linker_script,
             'object_files': self.resources.objects,
             'libraries': libraries,
             'symbols': self.get_symbols()
         }
-        ctx.update(self.progen_flags)
+        ctx.update(self.flags)
         self.gen_file('codered_%s_project.tmpl' % self.target.lower(), ctx, '.project')
         self.gen_file('codered_%s_cproject.tmpl' % self.target.lower(), ctx, '.cproject')

tools/export/coide.py

@@ -98,7 +98,7 @@ def generate(self):
             self.resources.linker_script = ''
 
         ctx = {
-            'name': self.program_name,
+            'name': self.project_name,
             'source_files': source_files,
             'header_files': header_files,
             'include_paths': self.resources.inc_dirs,
@@ -111,4 +111,4 @@ def generate(self):
         target = self.target.lower()
 
         # Project file
-        self.gen_file('coide_%s.coproj.tmpl' % target, ctx, '%s.coproj' % self.program_name)
+        self.gen_file('coide_%s.coproj.tmpl' % target, ctx, '%s.coproj' % self.project_name)

tools/export/ds5_5.py

@@ -52,7 +52,7 @@ def generate(self):
                 })
 
         ctx = {
-            'name': self.program_name,
+            'name': self.project_name,
             'include_paths': self.resources.inc_dirs,
             'scatter_file': self.resources.linker_script,
             'object_files': self.resources.objects + self.resources.libraries,

tools/export/e2studio.py

@@ -33,7 +33,7 @@ def generate(self):
             libraries.append(l[3:])
 
         ctx = {
-            'name': self.program_name,
+            'name': self.project_name,
             'include_paths': self.resources.inc_dirs,
             'linker_script': self.resources.linker_script,
 
@@ -44,4 +44,4 @@ def generate(self):
         self.gen_file('e2studio_%s_project.tmpl' % self.target.lower(), ctx, '.project')
         self.gen_file('e2studio_%s_cproject.tmpl' % self.target.lower(), ctx, '.cproject')
         self.gen_file('e2studio_%s_gdbinit.tmpl' % self.target.lower(), ctx, '.gdbinit')
-        self.gen_file('e2studio_launch.tmpl', ctx, '%s OpenOCD.launch' % self.program_name)
+        self.gen_file('e2studio_launch.tmpl', ctx, '%s OpenOCD.launch' % self.project_name)

tools/export/emblocks.py

@@ -60,7 +60,7 @@ def generate(self):
             self.resources.linker_script = ''
 
         ctx = {
-            'name': self.program_name,
+            'name': self.project_name,
             'target': self.target,
             'toolchain': self.toolchain.name,
             'source_files': source_files,
@@ -77,4 +77,4 @@ def generate(self):
         }
 
         # EmBlocks intermediate file template
-        self.gen_file('emblocks.eix.tmpl', ctx, '%s.eix' % self.program_name)
+        self.gen_file('emblocks.eix.tmpl', ctx, '%s.eix' % self.project_name)