Patches related mbed online build system support:

* added/improved global chroot support
* added RESPONSE_FILES flag to support optional response files (on linux the cmd param length is 2 megabytes). Default True
* added unified handling for archive and link response file (similar to includes)
* added COMPILE_C_AS_CPP flag to support compiling of c files as cpp. Default False
* added mbedToolchain.init() for post __init__ hooks
* added caching to mbedToolchain.need_update() to reduce IO hits
* added support to identify compiler warning/error column (supports ARMCC, GCC and IAR). Errors/warnings now report file@line,col
* added global TOOLCHAIN_PATHS which allows overriding/changing of the toolchain paths. Also simplified ARM-related paths
* added target.json to mbed library release (by @0xc0170)* migrated compile_worker() to utils.py for lightweight thread initialization
* improved run_cmd() performance by removing unnecessary check about the command being executed (should be checked once in the relevant toolchain instead)
* removed remnants of Goanna support (should be reimplemented as hooks to compile/link/archive instead)
* fixes for Python 2.7 compatibility (by @0xc0170)
* fixes for Exporters (by @0xc0170)

Author: screamerbg

hal/targets.json

@@ -88,7 +88,7 @@ def get_config(src_path, target, toolchain_name):
     config = Config(target, src_paths)
 
     # If the 'target' argument is a string, convert it to a target instance
-    if isinstance(target, str):
+    if isinstance(target, basestring):
         try:
             target = TARGET_MAP[target]
         except KeyError:
@@ -149,7 +149,7 @@ def build_project(src_path, build_path, target, toolchain_name,
     config = config or Config(target, src_paths)
 
     # If the 'target' argument is a string, convert it to a target instance
-    if isinstance(target, str):
+    if isinstance(target, basestring):
         try:
             target = TARGET_MAP[target]
         except KeyError:
@@ -287,7 +287,7 @@ def build_library(src_paths, build_path, target, toolchain_name,
     config = Config(target, src_paths)
 
     # If the 'target' argument is a string, convert it to a target instance
-    if isinstance(target, str):
+    if isinstance(target, basestring):
         try:
             target = TARGET_MAP[target]
         except KeyError:

tools/build_api.py

@@ -20,6 +20,7 @@
 from os.path import join, abspath, dirname, normpath
 from optparse import OptionParser
 import json
+from shutil import copy
 
 # Be sure that the tools directory is in the search path
 ROOT = abspath(join(dirname(__file__), ".."))
@@ -31,7 +32,7 @@
 from tools.test_exporters import ReportExporter, ResultExporterType
 from tools.test_api import SingleTestRunner
 from tools.test_api import singletest_in_cli_mode
-from tools.paths import TEST_DIR
+from tools.paths import TEST_DIR, MBED_LIBRARIES
 from tools.tests import TEST_MAP
 
 OFFICIAL_MBED_LIBRARY_BUILD = (
@@ -169,6 +170,9 @@
                 except Exception, e:
                     print str(e)
 
+    # copy targets.json file as part of the release
+    copy(join(dirname(abspath(__file__)), '..', 'hal', 'targets.json'), MBED_LIBRARIES)
+
     # Write summary of the builds
     if options.report_build_file_name:
         file_report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")

tools/build_release.py

@@ -182,7 +182,7 @@ def __init__(self, target, top_level_dirs = []):
         self.lib_config_data = {}
         # Make sure that each config is processed only once
         self.processed_configs = {}
-        self.target = target if isinstance(target, str) else target.name
+        self.target = target if isinstance(target, basestring) else target.name
         self.target_labels = Target.get_target(self.target).get_labels()
         self.added_features = set()
         self.removed_features = set()

tools/config.py

@@ -61,7 +61,7 @@ def __scan_and_copy(self, src_path, trg_path):
 
         for r_type in ['headers', 's_sources', 'c_sources', 'cpp_sources',
             'objects', 'libraries', 'linker_script',
-            'lib_builds', 'lib_refs', 'repo_files', 'hex_files', 'bin_files']:
+            'lib_builds', 'lib_refs', 'hex_files', 'bin_files']:
             r = getattr(resources, r_type)
             if r:
                 self.toolchain.copy_files(r, trg_path, resources=resources)
@@ -141,16 +141,21 @@ def scan_and_copy_resources(self, prj_paths, trg_path, relative=False):
         # Copy only the file for the required target and toolchain
         lib_builds = []
         # Create the configuration object
+        if isinstance(prj_paths, basestring):
+            prj_paths = [prj_paths]
         config = Config(self.target, prj_paths)
         for src in ['lib', 'src']:
-            resources = reduce(add, [self.__scan_and_copy(join(path, src), trg_path) for path in prj_paths])
+            resources = self.__scan_and_copy(join(prj_paths[0], src), trg_path)
+            for path in prj_paths[1:]:
+                resources.add(self.__scan_and_copy(join(path, src), trg_path))
+
             lib_builds.extend(resources.lib_builds)
 
             # The repository files
-            for repo_dir in resources.repo_dirs:
-                repo_files = self.__scan_all(repo_dir)
-                for path in proj_paths :
-                    self.toolchain.copy_files(repo_files, trg_path, rel_path=join(path, src))
+            #for repo_dir in resources.repo_dirs:
+            #    repo_files = self.__scan_all(repo_dir)
+            #    for path in prj_paths:
+            #        self.toolchain.copy_files(repo_files, trg_path, rel_path=join(path, src))
 
         # The libraries builds
         for bld in lib_builds:
@@ -178,19 +183,14 @@ def scan_and_copy_resources(self, prj_paths, trg_path, relative=False):
         # Loads the resources into the config system which might expand/modify resources based on config data
         self.resources = config.load_resources(resources)
 
-
         if hasattr(self, "MBED_CONFIG_HEADER_SUPPORTED") and self.MBED_CONFIG_HEADER_SUPPORTED :
             # Add the configuration file to the target directory
             self.config_header = self.toolchain.MBED_CONFIG_FILE_NAME
             config.get_config_data_header(join(trg_path, self.config_header))
             self.config_macros = []
-        else :
+        else:
             # And add the configuration macros to the toolchain
             self.config_macros = config.get_config_data_macros()
-        # Check the existence of a binary build of the mbed library for the desired target
-        # This prevents exporting the mbed libraries from source
-        # if not self.toolchain.mbed_libs:
-        #    raise OldLibrariesException()
 
     def gen_file(self, template_file, data, target_file):
         template_path = join(Exporter.TEMPLATE_DIR, template_file)

tools/export/exporters.py

@@ -19,7 +19,6 @@
 
 from tools.export.exporters import Exporter
 from tools.targets import TARGET_MAP, TARGET_NAMES
-from tools.settings import ARM_INC
 
 # If you wish to add a new target, add it to project_generator_definitions, and then
 # define progen_target name in the target class (`` self.progen_target = 'my_target_name' ``)
@@ -79,8 +78,6 @@ def generate(self, progen_build=False):
         project_data['tool_specific']['uvision']['misc']['c_flags'] = list(set(self.progen_flags['common_flags'] + self.progen_flags['c_flags'] + self.progen_flags['cxx_flags']))
         # not compatible with c99 flag set in the template
         project_data['tool_specific']['uvision']['misc']['c_flags'].remove("--c99")
-        # ARM_INC is by default as system inclusion, not required for exported project
-        project_data['tool_specific']['uvision']['misc']['c_flags'].remove("-I \""+ARM_INC+"\"")
         # cpp is not required as it's implicit for cpp files
         project_data['tool_specific']['uvision']['misc']['c_flags'].remove("--cpp")
         # we want no-vla for only cxx, but it's also applied for C in IDE, thus we remove it
@@ -98,7 +95,7 @@ def generate(self, progen_build=False):
                 project_data['common']['macros'].pop(i)
             i += 1
         project_data['common']['macros'].append('__ASSERT_MSG')
-        project_data['common']['build_dir'] = join(project_data['common']['build_dir'], 'uvision4')
+        project_data['common']['build_dir'] = project_data['common']['build_dir'] + '\\' + 'uvision4'
         if progen_build:
             self.progen_gen_file('uvision', project_data, True)
         else:

tools/export/uvision4.py

@@ -19,7 +19,6 @@
 
 from tools.export.exporters import Exporter
 from tools.targets import TARGET_MAP, TARGET_NAMES
-from tools.settings import ARM_INC
 
 # If you wish to add a new target, add it to project_generator_definitions, and then
 # define progen_target name in the target class (`` self.progen_target = 'my_target_name' ``)
@@ -81,8 +80,6 @@ def generate(self, progen_build=False):
         project_data['tool_specific']['uvision5']['misc']['asm_flags'] = list(set(self.progen_flags['asm_flags']))
         # cxx flags included, as uvision have them all in one tab
         project_data['tool_specific']['uvision5']['misc']['c_flags'] = list(set(self.progen_flags['common_flags'] + self.progen_flags['c_flags'] + self.progen_flags['cxx_flags']))
-        # ARM_INC is by default as system inclusion, not required for exported project
-        project_data['tool_specific']['uvision5']['misc']['c_flags'].remove("-I \""+ARM_INC+"\"")
         # not compatible with c99 flag set in the template
         project_data['tool_specific']['uvision5']['misc']['c_flags'].remove("--c99")
         # cpp is not required as it's implicit for cpp files
@@ -102,6 +99,7 @@ def generate(self, progen_build=False):
                 project_data['common']['macros'].pop(i)
             i += 1
         project_data['common']['macros'].append('__ASSERT_MSG')
+        project_data['common']['build_dir'] = project_data['common']['build_dir'] + '\\' + 'uvision5'
         if progen_build:
             self.progen_gen_file('uvision5', project_data, True)
         else:

tools/export/uvision5.py

@@ -80,17 +80,6 @@
             print "WARNING: MBED_%s set as environment variable but doesn't exist" % _n
 
 
-##############################################################################
-# ARM Compiler Paths
-##############################################################################
-
-ARM_BIN = join(ARM_PATH, "bin")
-ARM_INC = join(ARM_PATH, "include")
-ARM_LIB = join(ARM_PATH, "lib")
-ARM_CPPLIB = join(ARM_LIB, "cpplib")
-MY_ARM_CLIB = join(ARM_LIB, "lib", "microlib")
-
-
 ##############################################################################
 # Test System Settings
 ##############################################################################

tools/settings.py

@@ -122,7 +122,7 @@ def ignore_path(name, reg_exps):
 class MbedRepository:
     @staticmethod
     def run_and_print(command, cwd):
-        stdout, _, _ = run_cmd(command, wd=cwd, redirect=True)
+        stdout, _, _ = run_cmd(command, work_dir=cwd, redirect=True)
         print(stdout)
 
     def __init__(self, name, team = None):
@@ -147,7 +147,7 @@ def __init__(self, name, team = None):
     def publish(self):
         # The maintainer has to evaluate the changes first and explicitly accept them
         self.run_and_print(['hg', 'addremove'], cwd=self.path)
-        stdout, _, _ = run_cmd(['hg', 'status'], wd=self.path)
+        stdout, _, _ = run_cmd(['hg', 'status'], work_dir=self.path)
         if stdout == '':
             print "No changes"
             return False

tools/synch.py

@@ -22,7 +22,10 @@
     "Cortex-M1" : ["M1", "CORTEX_M", "LIKE_CORTEX_M1"],
     "Cortex-M3" : ["M3", "CORTEX_M", "LIKE_CORTEX_M3"],
     "Cortex-M4" : ["M4", "CORTEX_M", "RTOS_M4_M7", "LIKE_CORTEX_M4"],
+    "Cortex-M4F" : ["M4", "CORTEX_M", "RTOS_M4_M7", "LIKE_CORTEX_M4"],
     "Cortex-M7" : ["M7", "CORTEX_M", "RTOS_M4_M7", "LIKE_CORTEX_M7"],
+    "Cortex-M7F" : ["M7", "CORTEX_M", "RTOS_M4_M7", "LIKE_CORTEX_M7"],
+    "Cortex-M7FD" : ["M7", "CORTEX_M", "RTOS_M4_M7", "LIKE_CORTEX_M7"],
     "Cortex-A9" : ["A9", "CORTEX_A", "LIKE_CORTEX_A9"]
 }
 
@@ -58,6 +61,9 @@ class Target:
     # need to be computed differently than regular attributes
     __cumulative_attributes = ['extra_labels', 'macros', 'device_has', 'features']
 
+    # List of targets that were added dynamically using "add_py_targets" (see below)
+    __py_targets = set()
+
     # Location of the 'targets.json' file
     __targets_json_location = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'hal', 'targets.json')
 

tools/targets.py

@@ -58,7 +58,7 @@
 from tools.build_api import add_result_to_report
 from tools.build_api import scan_for_source_paths
 from tools.libraries import LIBRARIES, LIBRARY_MAP
-from tools.toolchains import TOOLCHAIN_BIN_PATH
+from tools.toolchains import TOOLCHAIN_PATHS
 from tools.toolchains import TOOLCHAINS
 from tools.test_exporters import ReportExporter, ResultExporterType
 from tools.utils import argparse_filestring_type
@@ -1343,8 +1343,8 @@ def print_test_configuration_from_json(json_data, join_delim=", "):
                 if conflict:
                     cell_val += '*'
                 # Check for conflicts: toolchain vs toolchain path
-                if toolchain in TOOLCHAIN_BIN_PATH:
-                    toolchain_path = TOOLCHAIN_BIN_PATH[toolchain]
+                if toolchain in TOOLCHAIN_PATHS:
+                    toolchain_path = TOOLCHAIN_PATHS[toolchain]
                     if not os.path.isdir(toolchain_path):
                         conflict_path = True
                         if toolchain not in toolchain_path_conflicts:
@@ -1368,8 +1368,8 @@ def print_test_configuration_from_json(json_data, join_delim=", "):
 
         for toolchain in toolchain_path_conflicts:
         # Let's check toolchain configuration
-            if toolchain in TOOLCHAIN_BIN_PATH:
-                toolchain_path = TOOLCHAIN_BIN_PATH[toolchain]
+            if toolchain in TOOLCHAIN_PATHS:
+                toolchain_path = TOOLCHAIN_PATHS[toolchain]
                 if not os.path.isdir(toolchain_path):
                     result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
     return result

tools/test_api.py

@@ -0,0 +1,2 @@
+<?xml version="1.0" ?>
+<testsuites/>

tools/test_reports/test_build_report_K64F.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" ?>
+<testsuites/>