diff --git a/internal/lzss/README.md b/internal/lzss/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/internal/lzss/lzss.c b/internal/lzss/lzss.c deleted file mode 100644 index 782a65e6..00000000 --- a/internal/lzss/lzss.c +++ /dev/null @@ -1,194 +0,0 @@ -/* LZSS encoder-decoder (Haruhiko Okumura; public domain) */ - -#include -#include - -#define EI 11 /* typically 10..13 */ -#define EJ 4 /* typically 4..5 */ -#define P 1 /* If match length <= P then output one character */ -#define N (1 << EI) /* buffer size */ -#define F ((1 << EJ) + 1) /* lookahead buffer size */ - -int bit_buffer = 0, bit_mask = 128; -unsigned long codecount = 0, textcount = 0; -unsigned char buffer[N * 2]; -FILE *infile, *outfile; - -void error(void) -{ - printf("Output error\n"); exit(1); -} - -void putbit1(void) -{ - bit_buffer |= bit_mask; - if ((bit_mask >>= 1) == 0) { - if (fputc(bit_buffer, outfile) == EOF) error(); - bit_buffer = 0; bit_mask = 128; codecount++; - } -} - -void putbit0(void) -{ - if ((bit_mask >>= 1) == 0) { - if (fputc(bit_buffer, outfile) == EOF) error(); - bit_buffer = 0; bit_mask = 128; codecount++; - } -} - -void flush_bit_buffer(void) -{ - if (bit_mask != 128) { - if (fputc(bit_buffer, outfile) == EOF) error(); - codecount++; - } -} - -void output1(int c) -{ - int mask; - - putbit1(); - mask = 256; - while (mask >>= 1) { - if (c & mask) putbit1(); - else putbit0(); - } -} - -void output2(int x, int y) -{ - int mask; - - putbit0(); - mask = N; - while (mask >>= 1) { - if (x & mask) putbit1(); - else putbit0(); - } - mask = (1 << EJ); - while (mask >>= 1) { - if (y & mask) putbit1(); - else putbit0(); - } -} - -void encode(void) -{ - int i, j, f1, x, y, r, s, bufferend, c; - - for (i = 0; i < N - F; i++) buffer[i] = ' '; - for (i = N - F; i < N * 2; i++) { - if ((c = fgetc(infile)) == EOF) break; - buffer[i] = c; textcount++; - } - bufferend = i; r = N - F; s = 0; - while (r < bufferend) { - f1 = (F <= bufferend - r) ? F : bufferend - r; - x = 0; y = 1; c = buffer[r]; - for (i = r - 1; i >= s; i--) - if (buffer[i] == c) { - for (j = 1; j < f1; j++) - if (buffer[i + j] != buffer[r + j]) break; - if (j > y) { - x = i; y = j; - } - } - if (y <= P) { y = 1; output1(c); } - else output2(x & (N - 1), y - 2); - r += y; s += y; - if (r >= N * 2 - F) { - for (i = 0; i < N; i++) buffer[i] = buffer[i + N]; - bufferend -= N; r -= N; s -= N; - while (bufferend < N * 2) { - if ((c = fgetc(infile)) == EOF) break; - buffer[bufferend++] = c; textcount++; - } - } - } - flush_bit_buffer(); -// printf("text: %ld bytes\n", textcount); -// printf("code: %ld bytes (%ld%%)\n", -// codecount, (codecount * 100) / textcount); -} - -int getbit(int n) /* get n bits */ -{ - int i, x; - static int buf, mask = 0; - - x = 0; - for (i = 0; i < n; i++) { - if (mask == 0) { - if ((buf = fgetc(infile)) == EOF) return EOF; - mask = 128; - } - x <<= 1; - if (buf & mask) x++; - mask >>= 1; - } - return x; -} - -void decode(void) -{ - int i, j, k, r, c; - - for (i = 0; i < N - F; i++) buffer[i] = ' '; - r = N - F; - while ((c = getbit(1)) != EOF) { - if (c) { - if ((c = getbit(8)) == EOF) break; - fputc(c, outfile); - buffer[r++] = c; r &= (N - 1); - } else { - if ((i = getbit(EI)) == EOF) break; - if ((j = getbit(EJ)) == EOF) break; - for (k = 0; k <= j + 1; k++) { - c = buffer[(i + k) & (N - 1)]; - fputc(c, outfile); - buffer[r++] = c; r &= (N - 1); - } - } - } -} - -int encode_file(char const * in, char const * out) -{ - // reset counters - bit_buffer = 0, bit_mask = 128; - codecount = 0, textcount = 0; - - infile = fopen(in, "rb"); - if (infile == NULL) return 0; - - outfile = fopen(out, "wb"); - if (outfile == NULL) return 0; - - encode(); - - fclose(infile); - fclose(outfile); - - return 0; -} - -int decode_file(char const * in, char const * out) -{ - // reset counters - bit_buffer = 0, bit_mask = 128; - codecount = 0, textcount = 0; - - infile = fopen(in, "rb"); - if (infile == NULL) return 0; - - outfile = fopen(out, "wb"); - if (outfile == NULL) return 0; - - decode(); - - fclose(infile); - fclose(outfile); - - return 0; -} diff --git a/internal/lzss/lzss.go b/internal/lzss/lzss.go index 897d8eb3..7aa7f6cb 100644 --- a/internal/lzss/lzss.go +++ b/internal/lzss/lzss.go @@ -1,3 +1,5 @@ +// This code is a go port of LZSS encoder-decoder (Haruhiko Okumura; public domain) +// // This file is part of arduino-cloud-cli. // // Copyright (C) 2021 ARDUINO SA (http://www.arduino.cc/) @@ -17,27 +19,190 @@ package lzss -// #cgo CFLAGS: -g -Wall -// #include -// #include "lzss.h" -import "C" import ( - // "fmt" - "sync" - "unsafe" + "bytes" +) + +const ( + idxsz = 11 // Size of buffer indexes in bits, typically 10..13 bits. + lensz = 4 // Size of lookahead indexes in bits, typically 4..5 bits. + + charsz = 8 // Size of encoded chars in bits. + bytemask = 128 // Mask with a bit in 8th position. Used to iterate through bits of a char. + + threshold = 1 // If match length > threshold then output a token (idx, len), otherwise output one char. + + bufsz = 1 << idxsz // Buffer size. + looksz = (1 << lensz) + 1 // Lookahead buffer size. + historysz = bufsz - looksz // History buffer size. + + charStartBit = true // Indicates next bits encode a char. + tokenStartBit = false // Indicates next bits encode a token. ) -func Encode(source, destination string) { +// Encode takes a slice of bytes, compresses it using the lzss compression algorithm +// and returns the result in a new bytes buffer. +func Encode(data []byte) []byte { + // buffer is made up of two parts: the first is for already processed data (history); the second is for new data + buffer := make([]byte, bufsz*2) + // Initialize the old-data part (history) of the buffer + for i := 0; i < historysz; i++ { + buffer[i] = ' ' + } + out := newResult() + in := newFiller(data) + + // Fill the new-data part of the buffer + n := in.fill(buffer[historysz:]) + bufferend := historysz + n + for current := historysz; current < bufferend; { + idx, len := findLargestMatch(buffer, current, bufferend) + if len <= threshold { + out.addChar(buffer[current]) + len = 1 + } else { + out.addToken(idx, len) + } + + current += len + if current >= bufsz*2-looksz { + // Shift processed bytes to the old-data portion of the buffer + copy(buffer[:bufsz], buffer[bufsz:]) + current -= bufsz + // Refill the new-data portion of the buffer + bufferend -= bufsz + bufferend += in.fill(buffer[bufferend:]) + } + } + + out.flush() + return out.bytes() +} + +func min(x, y int) int { + if x < y { + return x + } + return y +} + +// findLargestMatch looks for the largest sequence of characters (from current to current+ahead) +// contained in the history of the buffer. +// It returns the index of the found match, if any, and its length. +// The index is relative to the current position. If idx 0 is returned than no match has been found. +func findLargestMatch(buf []byte, current, size int) (idx, len int) { + idx = 0 + len = 1 + ahead := min(looksz, size-current) + history := current - historysz + c := buf[current] + for i := current - 1; i >= history; i-- { + if buf[i] == c { + var j int + for j = 1; j < ahead; j++ { + if buf[i+j] != buf[current+j] { + break + } + } + if j > len { + idx = i + len = j + } + } + } + return +} + +// filler abstracts the process of consuming an input buffer +// using its bytes to fill another buffer. +// It's been used to facilitate the handling of the input buffer in the Encode function. +type filler struct { + src []byte + idx int +} + +func newFiller(src []byte) *filler { + return &filler{ + src: src, + } +} + +// fill tries to fill all the dst buffer with bytes read from src. +// It returns the number of bytes moved from src to dst. +// The src buffer offset is then incremented so that all the content of src +// can be consumed in small chunks. +func (f *filler) fill(dst []byte) int { + n := copy(dst, f.src[f.idx:]) + f.idx += n + return n +} + +// result is responsible for storing the actual result of the encoding. +// It knows how to store characters and tokens in the resulting buffer. +// It must be flushed at the end of the encoding in order to store the +// remaining bits of bitBuffer. +type result struct { + bitBuffer int + bitMask int + out *bytes.Buffer +} + +func newResult() *result { + return &result{ + bitBuffer: 0, + bitMask: bytemask, + out: &bytes.Buffer{}, + } +} + +// addChar stores a char in the out buffer. +func (r *result) addChar(c byte) { + i := int(c) + r.putbit(charStartBit) + for mask := (1 << charsz) >> 1; mask != 0; mask = mask >> 1 { + b := (i & mask) != 0 + r.putbit(b) + } +} + +// addToken stores a token in the out buffer. +func (r *result) addToken(idx, len int) { + // Adjust idx and len to fit idxsz and lensz bits respectively + idx &= bufsz - 1 + len -= 2 - var mutex sync.Mutex + r.putbit(tokenStartBit) + for mask := (1 << idxsz) >> 1; mask != 0; mask = mask >> 1 { + b := idx&mask != 0 + r.putbit(b) + } - src := C.CString(source) - defer C.free(unsafe.Pointer(src)) + for mask := (1 << lensz) >> 1; mask != 0; mask = mask >> 1 { + b := len&mask != 0 + r.putbit(b) + } +} + +func (r *result) flush() { + if r.bitMask != bytemask { + r.out.WriteByte(byte(r.bitBuffer)) + } +} - dst := C.CString(destination) - defer C.free(unsafe.Pointer(dst)) +// putbit puts the passed bit (true -> 1; false -> 0) in the bitBuffer. +// When bitBuffer contains an entire byte it's written to the out buffer. +func (r *result) putbit(b bool) { + if b { + r.bitBuffer |= r.bitMask + } + r.bitMask = r.bitMask >> 1 + if r.bitMask == 0 { + r.out.WriteByte(byte(r.bitBuffer)) + r.bitBuffer = 0 + r.bitMask = bytemask + } +} - mutex.Lock() - C.encode_file(src, dst) - mutex.Unlock() +func (r *result) bytes() []byte { + return r.out.Bytes() } diff --git a/internal/lzss/lzss.h b/internal/lzss/lzss.h deleted file mode 100644 index a4a44622..00000000 --- a/internal/lzss/lzss.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _LZSS_H -#define _LZSS_H - -int encode_file(char const * in, char const * out); - -#endif \ No newline at end of file diff --git a/internal/lzss/lzss_test.go b/internal/lzss/lzss_test.go new file mode 100644 index 00000000..4a3996e7 --- /dev/null +++ b/internal/lzss/lzss_test.go @@ -0,0 +1,45 @@ +package lzss + +import ( + "bytes" + "io/ioutil" + "testing" +) + +func TestEncode(t *testing.T) { + tests := []struct { + name string + infile string + outfile string + }{ + { + name: "blink", + infile: "testdata/blink.bin", + outfile: "testdata/blink.lzss", + }, + { + name: "cloud sketch", + infile: "testdata/cloud.bin", + outfile: "testdata/cloud.lzss", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input, err := ioutil.ReadFile(tt.infile) + if err != nil { + t.Fatal("couldn't open test file") + } + + want, err := ioutil.ReadFile(tt.outfile) + if err != nil { + t.Fatal("couldn't open test file") + } + + got := Encode(input) + if !bytes.Equal(want, got) { + t.Error("encoding failed") + } + }) + } +} diff --git a/internal/lzss/testdata/blink.bin b/internal/lzss/testdata/blink.bin new file mode 100755 index 00000000..b3731ec4 Binary files /dev/null and b/internal/lzss/testdata/blink.bin differ diff --git a/internal/lzss/testdata/blink.lzss b/internal/lzss/testdata/blink.lzss new file mode 100644 index 00000000..9d23a667 Binary files /dev/null and b/internal/lzss/testdata/blink.lzss differ diff --git a/internal/lzss/testdata/cloud.bin b/internal/lzss/testdata/cloud.bin new file mode 100755 index 00000000..0bf2398a Binary files /dev/null and b/internal/lzss/testdata/cloud.bin differ diff --git a/internal/lzss/testdata/cloud.lzss b/internal/lzss/testdata/cloud.lzss new file mode 100644 index 00000000..92d549eb Binary files /dev/null and b/internal/lzss/testdata/cloud.lzss differ diff --git a/internal/lzss/testdata/lorem.lzss b/internal/lzss/testdata/lorem.lzss deleted file mode 100644 index 07d0070c..00000000 Binary files a/internal/lzss/testdata/lorem.lzss and /dev/null differ diff --git a/internal/lzss/testdata/lorem.txt b/internal/lzss/testdata/lorem.txt deleted file mode 100644 index c942efc7..00000000 --- a/internal/lzss/testdata/lorem.txt +++ /dev/null @@ -1,9 +0,0 @@ -Lorem ipsum dolor sit amet, consectetur adipiscing elit. Suspendisse ligula dui, imperdiet ut vulputate semper, sollicitudin ut eros. Aliquam erat volutpat. In hac habitasse platea dictumst. Nam non tortor sit amet mauris rutrum eleifend. Pellentesque vel justo nibh. Vivamus sem risus, pharetra eget egestas eget, venenatis a ipsum. Cras ultrices aliquam sagittis. Donec lacinia urna ac orci congue ut adipiscing dolor fringilla. Nullam nibh magna, bibendum vulputate ornare id, hendrerit et metus. Nullam dapibus neque quis mi laoreet molestie. Mauris et dui lacus, sit amet egestas purus. - -Donec accumsan elementum accumsan. Nullam gravida dictum diam non semper. Curabitur vel magna in velit accumsan pulvinar eget in lorem. Duis vitae ante velit, at hendrerit nibh. Pellentesque lacus urna, cursus ac semper sagittis, viverra at sem. Quisque ullamcorper odio dolor. In quis pretium lacus. Maecenas lacinia urna id massa congue blandit. Suspendisse dapibus eros sit amet neque fermentum imperdiet. Cras interdum pulvinar eleifend. Suspendisse molestie neque a risus imperdiet convallis. In interdum dignissim pharetra. Morbi lectus tortor, pulvinar quis eleifend in, placerat at risus. Sed aliquam diam at metus adipiscing blandit. - -Integer tristique metus vel ipsum pulvinar dignissim quis vel quam. Donec auctor aliquet bibendum. Morbi aliquet malesuada ultrices. Vivamus ac leo odio. Nam tristique eros non arcu porttitor non volutpat mauris tempus. Proin vestibulum suscipit pretium. Etiam elit tortor, dictum a gravida porta, congue id dolor. Duis eget est vitae elit facilisis blandit. Proin tincidunt felis et ipsum pharetra tempor. Fusce imperdiet vulputate magna, vel lacinia neque volutpat a. Vivamus a elit dolor. Aliquam sollicitudin dui et leo elementum mattis. Quisque suscipit, lorem id eleifend imperdiet, ipsum lorem pharetra purus, vel tempus lectus ligula id tortor. Morbi eget eros vel sapien scelerisque aliquam pellentesque sed turpis. Duis vel lorem non eros semper fringilla vitae vitae erat. - -Vivamus porttitor pulvinar tristique. Proin sed elit ipsum. Phasellus faucibus pulvinar dapibus. Praesent quis sem in purus ultrices imperdiet. Aenean ut nulla urna. In tristique tincidunt urna, nec adipiscing velit laoreet ut. Curabitur et ante sed libero tristique pellentesque. Quisque porttitor sodales ipsum ut rhoncus. Nunc vitae diam gravida orci aliquam cursus vitae ut sapien. Proin ullamcorper felis eu nulla dapibus nec faucibus odio hendrerit. Aenean lorem magna, fermentum in tristique sit amet, accumsan ut massa. Fusce tristique, lectus rhoncus commodo sagittis, ligula felis consequat arcu, id pretium enim dolor id mi. Donec facilisis pulvinar luctus. Pellentesque vitae condimentum risus. Nam quis elit a orci adipiscing bibendum. - -Ut quis felis lorem, dignissim varius turpis. Sed convallis dui semper mauris fermentum porta. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Aliquam erat volutpat. Proin lorem felis, scelerisque nec commodo non, porta eu metus. Nulla id augue a turpis mollis pellentesque. Aenean in lectus et leo tincidunt auctor eu interdum est. Nulla varius, lorem congue laoreet laoreet, felis quam ullamcorper ligula, non pellentesque dui ipsum quis odio. Etiam sit amet blandit leo. Aenean venenatis molestie eros, in fermentum ipsum dictum eget. Donec ultricies feugiat nisl, non molestie mi congue quis. Quisque mattis augue nec neque fringilla varius. Proin sollicitudin risus et elit pretium congue. Sed consequat eros sit amet felis pulvinar pulvinar. Morbi in turpis eu nulla cursus venenatis at ut urna. Donec vel lectus quis nisi aliquam varius. \ No newline at end of file diff --git a/internal/ota/encoder.go b/internal/ota/encoder.go index 7f28e3f6..a9ab891e 100644 --- a/internal/ota/encoder.go +++ b/internal/ota/encoder.go @@ -22,9 +22,6 @@ import ( "encoding/binary" "hash/crc32" "io" - "io/ioutil" - "log" - "os" "strconv" "github.com/arduino/arduino-cloud-cli/internal/lzss" @@ -86,10 +83,7 @@ func (e *encoder) Write(binaryData []byte) (int, error) { } // Compress the compiled binary - compressed, err := e.compress(&binaryData) - if err != nil { - return 0, err - } + compressed := lzss.Encode(binaryData) // Prepend magic number and version field to payload var binDataComplete []byte @@ -98,12 +92,12 @@ func (e *encoder) Write(binaryData []byte) (int, error) { binDataComplete = append(binDataComplete, compressed...) //log.Println("binDataComplete is", len(binDataComplete), "bytes length") - headerSize, err := e.writeHeader(&binDataComplete) + headerSize, err := e.writeHeader(binDataComplete) if err != nil { return headerSize, err } - payloadSize, err := e.writePayload(&binDataComplete) + payloadSize, err := e.writePayload(binDataComplete) if err != nil { return payloadSize, err } @@ -117,23 +111,19 @@ func (e *encoder) Close() error { return e.w.Flush() } -func (e *encoder) writeHeader(binDataComplete *[]byte) (int, error) { +func (e *encoder) writeHeader(binDataComplete []byte) (int, error) { - // // Write the length of the content - // lengthAsBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(lengthAsBytes, uint32(len(*binDataComplete))) + binary.LittleEndian.PutUint32(lengthAsBytes, uint32(len(binDataComplete))) n, err := e.w.Write(lengthAsBytes) if err != nil { return n, err } - // // Calculate the checksum for binDataComplete - // - crc := crc32.ChecksumIEEE(*binDataComplete) + crc := crc32.ChecksumIEEE(binDataComplete) // encode the checksum uint32 value as 4 bytes crcAsBytes := make([]byte, 4) @@ -147,52 +137,6 @@ func (e *encoder) writeHeader(binDataComplete *[]byte) (int, error) { return len(lengthAsBytes) + len(crcAsBytes), nil } -func (e *encoder) writePayload(data *[]byte) (int, error) { - - // write the payload - payloadSize, err := e.w.Write(*data) - if err != nil { - return payloadSize, err - } - - return payloadSize, nil -} - -func (e *encoder) compress(data *[]byte) ([]byte, error) { - - // create a tmp file for input - inputFile, err := ioutil.TempFile("", "ota-lzss-input") - if err != nil { - log.Fatal(err) - return nil, err - } - defer os.Remove(inputFile.Name()) - - // create a tmp file for output - outputFile, err := ioutil.TempFile("", "ota-lzss-output") - if err != nil { - log.Fatal(err) - return nil, err - } - defer os.Remove(outputFile.Name()) - - // write data in the input file - ioutil.WriteFile(inputFile.Name(), *data, 644) - if err != nil { - log.Fatal(err) - return nil, err - } - - // Compress the binary data using LZSS - lzss.Encode(inputFile.Name(), outputFile.Name()) - - // reads compressed data from output file and write it into - // the writer - compressed, err := ioutil.ReadFile(outputFile.Name()) - if err != nil { - log.Fatal(err) - return nil, err - } - - return compressed, nil +func (e *encoder) writePayload(data []byte) (int, error) { + return e.w.Write(data) }