diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml index e11d36ed..8969a1bb 100644 --- a/.github/workflows/test-go.yml +++ b/.github/workflows/test-go.yml @@ -5,11 +5,15 @@ on: push: paths: - ".github/workflows/test-go.yml" + - "go.mod" + - "go.sum" - "Taskfile.yml" - "**.go" pull_request: paths: - ".github/workflows/test-go.yml" + - "go.mod" + - "go.sum" - "Taskfile.yml" - "**.go" workflow_dispatch: @@ -34,9 +38,6 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} version: 3.x - - name: Configure Git for go get access to private repo - run: git config --global url."https://${{ secrets.REPO_SCOPE_TOKEN }}:x-oauth-basic@github.com/".insteadOf "https://github.com/" - - name: Build application run: task go:build @@ -65,8 +66,5 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} version: 3.x - - name: Configure Git for go get access to private repo - run: git config --global url."https://${{ secrets.REPO_SCOPE_TOKEN }}:x-oauth-basic@github.com/".insteadOf "https://github.com/" - - name: Run tests run: task go:test diff --git a/.gitignore b/.gitignore index 26c068f4..0e923adc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ token -bin +coverage_*.txt libraries-repository-engine -src/code.google.com -src/github.com +libraries-repository-engine.exe +repository +repository.exe test/arduino/cc/repository/libraries/testdata/test_db.json config.json repos.txt diff --git a/README.md b/README.md index ff2599fe..f77c114f 100644 --- a/README.md +++ b/README.md @@ -2,29 +2,7 @@ BUILD ---------------------------- ``` -go get github.com/google/go-github/github -go get github.com/vaughan0/go-ini -go get github.com/blang/semver -go get github.com/stretchr/testify -go get github.com/arduino/arduino-modules/git - -go build arduino.cc/repository/libraries-repository-engine -``` - -You may want to setup git to allow "go get" from private repos, in this case you must -generate a personal access token from github that grants "repo access" permissions and do: - -``` -git config --global url."https://YOUR_ACCESS_TOKEN:x-oauth-basic@github.com/".insteadOf "https://github.com/" -``` - -the configuration will be saved inside `~/.gitconfig` as: - -``` -... -[url "https://YOUR_ACCESS_TOKEN:x-oauth-basic@github.com/"] - insteadOf = https://github.com/ -... +task go:build ``` TDD @@ -33,7 +11,7 @@ TDD In order to run the tests, type ``` -go test -v ./src/arduino.cc/repository/libraries/test/... +task go:test ``` RUN @@ -41,4 +19,4 @@ RUN Create a `config.json` file (or edit example one). Same thing for `repos.txt` file. -Run with `go run sync_libraries.go` or `go build` and then `./libraries-repository-engine` +Run with `go run sync_libraries.go` or `task go:build` and then `./libraries-repository-engine` diff --git a/Taskfile.yml b/Taskfile.yml index 56ad496a..6966653a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -2,29 +2,16 @@ version: "3" env: - GOPATH: "$(echo $PWD)" + DEFAULT_GO_PACKAGES: + sh: echo $(go list ./... | tr '\n' ' ') tasks: - go:deps: - desc: Install dependencies - cmds: - - go get github.com/arduino/arduino-modules/git - - go get github.com/arduino/golang-concurrent-workers - - go get github.com/blang/semver - - go get github.com/google/go-github/github - - go get github.com/stretchr/testify - - go get github.com/vaughan0/go-ini - go:build: desc: Build the project - deps: - - task: go:deps cmds: - - go build arduino.cc/repository/libraries-repository-engine + - go build -v -o libraries-repository-engine{{exeExt}} go:test: desc: Run unit tests - deps: - - task: go:deps cmds: - - go test -v ./src/arduino.cc/repository/libraries/... + - go test -v -short -run '{{default ".*" .GO_TEST_REGEX}}' {{default "-timeout 10m -coverpkg=./... -covermode=atomic" .GO_TEST_FLAGS}} -coverprofile=coverage_unit.txt {{default .DEFAULT_GO_PACKAGES .GO_PACKAGES}} diff --git a/fmt_fix_vet b/fmt_fix_vet index 3d6622b1..793c565b 100755 --- a/fmt_fix_vet +++ b/fmt_fix_vet @@ -1,8 +1,8 @@ #!/bin/bash -x -go fmt ./src/arduino.cc/repository/... +go fmt ./... go fmt sync_libraries.go -go fix ./src/arduino.cc/repository/... +go fix ./... go fix sync_libraries.go -go vet ./src/arduino.cc/repository/... +go vet ./... go vet sync_libraries.go diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..22bacee5 --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module arduino.cc/repository + +go 1.14 + +require ( + github.com/arduino/golang-concurrent-workers v0.0.0-20170202182617-6710cdc954bc + github.com/blang/semver v3.5.1+incompatible + github.com/go-git/go-git/v5 v5.3.0 + github.com/google/go-github v17.0.0+incompatible + github.com/google/go-querystring v1.1.0 // indirect + github.com/stretchr/testify v1.7.0 + github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..e44431f5 --- /dev/null +++ b/go.sum @@ -0,0 +1,110 @@ +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/arduino/golang-concurrent-workers v0.0.0-20170202182617-6710cdc954bc h1:PzGY1Ppud/Ng+LFHU16oOrWhYsnSLYurwiHlbVc/FJ0= +github.com/arduino/golang-concurrent-workers v0.0.0-20170202182617-6710cdc954bc/go.mod h1:E+WBbLkFBdPp+N+yijgbdDI33mr5pm6j42RYLN5K4do= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.1.0 h1:4pl5BV4o7ZG/lterP4S6WzJ6xr49Ba5ET9ygheTYahk= +github.com/go-git/go-billy/v5 v5.1.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12 h1:PbKy9zOy4aAKrJ5pibIRpVO2BXnK1Tlcg+caKI7Ox5M= +github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw= +github.com/go-git/go-git/v5 v5.3.0 h1:8WKMtJR2j8RntEXR/uvTKagfEt4GYlwQ7mntE4+0GWc= +github.com/go-git/go-git/v5 v5.3.0/go.mod h1:xdX4bWJ48aOrdhnl2XqHYstHbbp6+LFS4r4X+lNVprw= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks= +github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw= +github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210326060303-6b1517762897 h1:KrsHThm5nFk34YtATK1LsThyGhGbGe1olrte/HInHvs= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/arduino.cc/repository/libraries/bad_file_cleaner.go b/libraries/bad_file_cleaner.go similarity index 100% rename from src/arduino.cc/repository/libraries/bad_file_cleaner.go rename to libraries/bad_file_cleaner.go diff --git a/src/arduino.cc/repository/libraries/bad_file_cleaner_test.go b/libraries/bad_file_cleaner_test.go similarity index 100% rename from src/arduino.cc/repository/libraries/bad_file_cleaner_test.go rename to libraries/bad_file_cleaner_test.go diff --git a/src/arduino.cc/repository/libraries/clamav.go b/libraries/clamav.go similarity index 100% rename from src/arduino.cc/repository/libraries/clamav.go rename to libraries/clamav.go diff --git a/src/arduino.cc/repository/libraries/cron/fill_missing_checksums.go b/libraries/cron/fill_missing_checksums.go similarity index 100% rename from src/arduino.cc/repository/libraries/cron/fill_missing_checksums.go rename to libraries/cron/fill_missing_checksums.go diff --git a/src/arduino.cc/repository/libraries/db/db.go b/libraries/db/db.go similarity index 100% rename from src/arduino.cc/repository/libraries/db/db.go rename to libraries/db/db.go diff --git a/src/arduino.cc/repository/libraries/db/dependencies_test.go b/libraries/db/dependencies_test.go similarity index 100% rename from src/arduino.cc/repository/libraries/db/dependencies_test.go rename to libraries/db/dependencies_test.go diff --git a/src/arduino.cc/repository/libraries/db/index.go b/libraries/db/index.go similarity index 100% rename from src/arduino.cc/repository/libraries/db/index.go rename to libraries/db/index.go diff --git a/src/arduino.cc/repository/libraries/db/library.go b/libraries/db/library.go similarity index 100% rename from src/arduino.cc/repository/libraries/db/library.go rename to libraries/db/library.go diff --git a/src/arduino.cc/repository/libraries/db/versioning.go b/libraries/db/versioning.go similarity index 100% rename from src/arduino.cc/repository/libraries/db/versioning.go rename to libraries/db/versioning.go diff --git a/src/arduino.cc/repository/libraries/db/versioning_test.go b/libraries/db/versioning_test.go similarity index 100% rename from src/arduino.cc/repository/libraries/db/versioning_test.go rename to libraries/db/versioning_test.go diff --git a/src/arduino.cc/repository/libraries/file/SCCS.go b/libraries/file/SCCS.go similarity index 100% rename from src/arduino.cc/repository/libraries/file/SCCS.go rename to libraries/file/SCCS.go diff --git a/src/arduino.cc/repository/libraries/git_integration_test.go b/libraries/git_integration_test.go similarity index 82% rename from src/arduino.cc/repository/libraries/git_integration_test.go rename to libraries/git_integration_test.go index ef889a50..6b9df961 100644 --- a/src/arduino.cc/repository/libraries/git_integration_test.go +++ b/libraries/git_integration_test.go @@ -7,6 +7,7 @@ import ( "testing" "arduino.cc/repository/libraries/db" + "github.com/go-git/go-git/v5" "github.com/stretchr/testify/require" ) @@ -33,11 +34,14 @@ func TestUpdateLibraryJson(t *testing.T) { defer os.RemoveAll(r.FolderPath) - tags, err := r.ListTags() + tags, err := r.Repository.Tags() + require.NoError(t, err) + tag, err := tags.Next() require.NoError(t, err) - require.NotEmpty(t, tags) - err = r.CheckoutTag(tags[0]) + repoTree, err := r.Repository.Worktree() + require.NoError(t, err) + err = repoTree.Checkout(&git.CheckoutOptions{Hash: tag.Hash()}) require.NoError(t, err) library, err := GenerateLibraryFromRepo(r) diff --git a/src/arduino.cc/repository/libraries/github_release_downloader.go b/libraries/github_release_downloader.go similarity index 100% rename from src/arduino.cc/repository/libraries/github_release_downloader.go rename to libraries/github_release_downloader.go diff --git a/src/arduino.cc/repository/libraries/github_release_downloader_test.go b/libraries/github_release_downloader_test.go similarity index 100% rename from src/arduino.cc/repository/libraries/github_release_downloader_test.go rename to libraries/github_release_downloader_test.go diff --git a/src/arduino.cc/repository/libraries/hash/checksumhelper.go b/libraries/hash/checksumhelper.go similarity index 100% rename from src/arduino.cc/repository/libraries/hash/checksumhelper.go rename to libraries/hash/checksumhelper.go diff --git a/src/arduino.cc/repository/libraries/metadata/metadata.go b/libraries/metadata/metadata.go similarity index 100% rename from src/arduino.cc/repository/libraries/metadata/metadata.go rename to libraries/metadata/metadata.go diff --git a/src/arduino.cc/repository/libraries/metadata/metadata_test.go b/libraries/metadata/metadata_test.go similarity index 100% rename from src/arduino.cc/repository/libraries/metadata/metadata_test.go rename to libraries/metadata/metadata_test.go diff --git a/src/arduino.cc/repository/libraries/metadata/utils.go b/libraries/metadata/utils.go similarity index 100% rename from src/arduino.cc/repository/libraries/metadata/utils.go rename to libraries/metadata/utils.go diff --git a/src/arduino.cc/repository/libraries/metadata/utils_test.go b/libraries/metadata/utils_test.go similarity index 100% rename from src/arduino.cc/repository/libraries/metadata/utils_test.go rename to libraries/metadata/utils_test.go diff --git a/src/arduino.cc/repository/libraries/repoarchive.go b/libraries/repoarchive.go similarity index 100% rename from src/arduino.cc/repository/libraries/repoarchive.go rename to libraries/repoarchive.go diff --git a/src/arduino.cc/repository/libraries/repoclone.go b/libraries/repoclone.go similarity index 61% rename from src/arduino.cc/repository/libraries/repoclone.go rename to libraries/repoclone.go index 283c46dc..0fff9931 100644 --- a/src/arduino.cc/repository/libraries/repoclone.go +++ b/libraries/repoclone.go @@ -12,38 +12,59 @@ import ( "fmt" "arduino.cc/repository/libraries/metadata" - "github.com/arduino/arduino-modules/git" + "github.com/go-git/go-git/v5" ) -func CloneOrFetch(repoMeta *Repo, folderName string) (*git.Repository, error) { - var repo *git.Repository +// Repository represents a Git repository located on the filesystem. +type Repository struct { + Repository *git.Repository + FolderPath string + URL string +} + +func CloneOrFetch(repoMeta *Repo, folderName string) (*Repository, error) { + repo := Repository{ + FolderPath: folderName, + URL: repoMeta.Url, + } + if _, err := os.Stat(folderName); os.IsNotExist(err) { - repo, err = git.Clone(repoMeta.Url, folderName) + repo.Repository, err = git.PlainClone(folderName, false, &git.CloneOptions{URL: repoMeta.Url}) if err != nil { return nil, err } } else { - repo = &git.Repository{FolderPath: folderName, URL: repoMeta.Url} + repo.Repository, err = git.PlainOpen(folderName) + if err != nil { + return nil, err + } } - tags, err := repo.ListTags() + tags, err := repo.Repository.Tags() if err != nil { return nil, err } - for _, tag := range tags { - if err = repo.RemoveTag(tag); err != nil { + + for { + tag, err := tags.Next() + if err != nil { + // Reached end of tags + break + } + + if err = repo.Repository.DeleteTag(strings.TrimPrefix(tag.Name().String(), "refs/tags/")); err != nil { return nil, err } } - if err = repo.Fetch(); err != nil { + if err = repo.Repository.Fetch(&git.FetchOptions{Tags: git.AllTags}); err != nil { return nil, err } - return repo, err + return &repo, err } -func GenerateLibraryFromRepo(repo *git.Repository) (*metadata.LibraryMetadata, error) { +func GenerateLibraryFromRepo(repo *Repository) (*metadata.LibraryMetadata, error) { bytes, err := ioutil.ReadFile(filepath.Join(repo.FolderPath, "library.properties")) if err != nil { return nil, fmt.Errorf("can't read library.properties: %s", err) diff --git a/src/arduino.cc/repository/libraries/repoclone_test.go b/libraries/repoclone_test.go similarity index 100% rename from src/arduino.cc/repository/libraries/repoclone_test.go rename to libraries/repoclone_test.go diff --git a/src/arduino.cc/repository/libraries/repolist.go b/libraries/repolist.go similarity index 100% rename from src/arduino.cc/repository/libraries/repolist.go rename to libraries/repolist.go diff --git a/src/arduino.cc/repository/libraries/repolist_test.go b/libraries/repolist_test.go similarity index 100% rename from src/arduino.cc/repository/libraries/repolist_test.go rename to libraries/repolist_test.go diff --git a/src/arduino.cc/repository/libraries/testdata/git_only_servo.txt b/libraries/testdata/git_only_servo.txt similarity index 100% rename from src/arduino.cc/repository/libraries/testdata/git_only_servo.txt rename to libraries/testdata/git_only_servo.txt diff --git a/src/arduino.cc/repository/libraries/testdata/git_test_repos.txt b/libraries/testdata/git_test_repos.txt similarity index 100% rename from src/arduino.cc/repository/libraries/testdata/git_test_repos.txt rename to libraries/testdata/git_test_repos.txt diff --git a/src/arduino.cc/repository/libraries/testdata/lib_valid/test.txt b/libraries/testdata/lib_valid/test.txt similarity index 100% rename from src/arduino.cc/repository/libraries/testdata/lib_valid/test.txt rename to libraries/testdata/lib_valid/test.txt diff --git a/src/arduino.cc/repository/libraries/testdata/lib_with_exe/fake.exe b/libraries/testdata/lib_with_exe/fake.exe similarity index 100% rename from src/arduino.cc/repository/libraries/testdata/lib_with_exe/fake.exe rename to libraries/testdata/lib_with_exe/fake.exe diff --git a/src/arduino.cc/repository/libraries/testdata/lib_with_exe/test.txt b/libraries/testdata/lib_with_exe/test.txt similarity index 100% rename from src/arduino.cc/repository/libraries/testdata/lib_with_exe/test.txt rename to libraries/testdata/lib_with_exe/test.txt diff --git a/src/arduino.cc/repository/libraries/testdata/lib_with_forbidden_file/.development b/libraries/testdata/lib_with_forbidden_file/.development similarity index 100% rename from src/arduino.cc/repository/libraries/testdata/lib_with_forbidden_file/.development rename to libraries/testdata/lib_with_forbidden_file/.development diff --git a/src/arduino.cc/repository/libraries/testdata/lib_with_forbidden_file/test.txt b/libraries/testdata/lib_with_forbidden_file/test.txt similarity index 100% rename from src/arduino.cc/repository/libraries/testdata/lib_with_forbidden_file/test.txt rename to libraries/testdata/lib_with_forbidden_file/test.txt diff --git a/src/arduino.cc/repository/libraries/zip/testzip/.hiddenfile b/libraries/zip/testzip/.hiddenfile similarity index 100% rename from src/arduino.cc/repository/libraries/zip/testzip/.hiddenfile rename to libraries/zip/testzip/.hiddenfile diff --git a/src/arduino.cc/repository/libraries/zip/testzip/.svn/.gitkeep b/libraries/zip/testzip/.svn/.gitkeep similarity index 100% rename from src/arduino.cc/repository/libraries/zip/testzip/.svn/.gitkeep rename to libraries/zip/testzip/.svn/.gitkeep diff --git a/src/arduino.cc/repository/libraries/zip/testzip/CVS/.gitkeep b/libraries/zip/testzip/CVS/.gitkeep similarity index 100% rename from src/arduino.cc/repository/libraries/zip/testzip/CVS/.gitkeep rename to libraries/zip/testzip/CVS/.gitkeep diff --git a/src/arduino.cc/repository/libraries/zip/testzip/testfile.txt b/libraries/zip/testzip/testfile.txt similarity index 100% rename from src/arduino.cc/repository/libraries/zip/testzip/testfile.txt rename to libraries/zip/testzip/testfile.txt diff --git a/src/arduino.cc/repository/libraries/zip/testzip/testfolder/CVS/.gitkeep b/libraries/zip/testzip/testfolder/CVS/.gitkeep similarity index 100% rename from src/arduino.cc/repository/libraries/zip/testzip/testfolder/CVS/.gitkeep rename to libraries/zip/testzip/testfolder/CVS/.gitkeep diff --git a/src/arduino.cc/repository/libraries/zip/testzip/testfolder/testfileinfolder.txt b/libraries/zip/testzip/testfolder/testfileinfolder.txt similarity index 100% rename from src/arduino.cc/repository/libraries/zip/testzip/testfolder/testfileinfolder.txt rename to libraries/zip/testzip/testfolder/testfileinfolder.txt diff --git a/src/arduino.cc/repository/libraries/zip/ziphelper.go b/libraries/zip/ziphelper.go similarity index 100% rename from src/arduino.cc/repository/libraries/zip/ziphelper.go rename to libraries/zip/ziphelper.go diff --git a/src/arduino.cc/repository/libraries/zip/ziphelper_test.go b/libraries/zip/ziphelper_test.go similarity index 100% rename from src/arduino.cc/repository/libraries/zip/ziphelper_test.go rename to libraries/zip/ziphelper_test.go diff --git a/setup_go_env_vars b/setup_go_env_vars deleted file mode 100644 index 91718b72..00000000 --- a/setup_go_env_vars +++ /dev/null @@ -1,3 +0,0 @@ -export PATH=$PATH:/usr/local/go/bin/ -export GOPATH=`pwd` -export GOROOT=/usr/local/go diff --git a/src/.gitignore b/src/.gitignore deleted file mode 100644 index c7643418..00000000 --- a/src/.gitignore +++ /dev/null @@ -1 +0,0 @@ -github.com/arduino diff --git a/src/arduino.cc/repository/libraries-repository-engine/sync_libraries.go b/sync_libraries.go similarity index 92% rename from src/arduino.cc/repository/libraries-repository-engine/sync_libraries.go rename to sync_libraries.go index 2d51175c..b4c520e1 100644 --- a/src/arduino.cc/repository/libraries-repository-engine/sync_libraries.go +++ b/sync_libraries.go @@ -12,8 +12,9 @@ import ( "arduino.cc/repository/libraries" "arduino.cc/repository/libraries/db" "arduino.cc/repository/libraries/hash" - "github.com/arduino/arduino-modules/git" cc "github.com/arduino/golang-concurrent-workers" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" ) type Config struct { @@ -202,13 +203,19 @@ func syncLibrary(logger *log.Logger, repoMetadata *libraries.Repo, libraryDb *db } // Retrieve the list of git-tags - tags, err := repo.ListTags() + tags, err := repo.Repository.Tags() if err != nil { logger.Printf("Error retrieving git-tags: %s", err) return } - for _, tag := range tags { + for { + tag, err := tags.Next() + if err != nil { + // Reached end of tags + break + } + // Sync the library release for each git-tag err = syncLibraryTaggedRelease(logger, repo, tag, repoMetadata, libraryDb) if err != nil { @@ -217,11 +224,16 @@ func syncLibrary(logger *log.Logger, repoMetadata *libraries.Repo, libraryDb *db } } -func syncLibraryTaggedRelease(logger *log.Logger, repo *git.Repository, tag string, repoMeta *libraries.Repo, libraryDb *db.DB) error { +func syncLibraryTaggedRelease(logger *log.Logger, repo *libraries.Repository, tag *plumbing.Reference, repoMeta *libraries.Repo, libraryDb *db.DB) error { // Checkout desired tag - logger.Printf("Checking out tag: %s", tag) - if out, err := repo.CheckoutTagWithOutput(tag); err != nil { - logger.Printf("git output: %s", out) + logger.Printf("Checking out tag: %s", tag.Name()) + + repoTree, err := repo.Repository.Worktree() + if err != nil { + return err + } + + if err = repoTree.Checkout(&git.CheckoutOptions{Hash: tag.Hash()}); err != nil { return fmt.Errorf("Error checking out repo: %s", err) }