Skip to content

fix: allow mtime and mode to be optional #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Dec 19, 2019
38 changes: 20 additions & 18 deletions src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,6 @@ function Data (type, data) {
this.data = data
this.blockSizes = []

if (this.type === 'file') {
this.mode = parseInt('0644', 8)
}

if (this.type === 'directory' || this.type === 'hamt-sharded-directory') {
this.mode = parseInt('0755', 8)
}

this.addBlockSize = (size) => {
this.blockSizes.push(size)
}
Expand Down Expand Up @@ -92,12 +84,20 @@ function Data (type, data) {
blockSizes = undefined
}

if ((this.type === 'directory' || this.type === 'hamt-sharded-directory') && this.mode === parseInt('0755', 8)) {
delete this.mode
let mode

if (!isNaN(this.mode)) {
mode = {
value: this.mode
}
}

if (this.type === 'file' && this.mode === parseInt('0644', 8)) {
delete this.mode
let mtime

if (this.mtime) {
mtime = {
seconds: Math.round(this.mtime.getTime() / 1000)
}
}

return unixfsData.encode({
Expand All @@ -107,27 +107,29 @@ function Data (type, data) {
blocksizes: blockSizes,
hashType: this.hashType,
fanout: this.fanout,
mode: this.mode,
mtime: this.mtime
mode: mode,
mtime: mtime
})
}
}

// decode from protobuf https://github.com/ipfs/go-ipfs/blob/master/unixfs/format.go#L24
Data.unmarshal = (marsheled) => {
const decoded = unixfsData.decode(marsheled)
Data.unmarshal = (marshaled) => {
const decoded = unixfsData.decode(marshaled)

if (!decoded.Data) {
decoded.Data = undefined
}

const obj = new Data(types[decoded.Type], decoded.Data)
obj.blockSizes = decoded.blocksizes

if (decoded.mode) {
obj.mode = decoded.mode
obj.mode = decoded.mode.value

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As per ipfs/specs#231, this needs to mask off with 07777

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's not in the spec yet, will do a further PR if it gets added.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

However, let's be careful. If we get a file that has extra bits, we should round-trip those bits. Otherwise, it's hard to add features later.

}

if (decoded.mtime) {
obj.mtime = decoded.mtime
obj.mtime = new Date(decoded.mtime.seconds * 1000)
}

return obj
Expand Down
14 changes: 11 additions & 3 deletions src/unixfs.proto.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,18 @@ module.exports = `message Data {
repeated uint64 blocksizes = 4;
optional uint64 hashType = 5;
optional uint64 fanout = 6;
optional uint32 mode = 7;
optional int64 mtime = 8;
optional Mode mode = 7;
optional Mtime mtime = 8;
}

message Metadata {
optional string MimeType = 1;
required string MimeType = 1;
}

message Mode {
required uint32 value = 1;
}

message Mtime {
required int64 seconds = 1;
}`
195 changes: 106 additions & 89 deletions test/unixfs-format.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,121 +17,138 @@ const Buffer = require('safe-buffer').Buffer
describe('unixfs-format', () => {
it('raw', () => {
const data = new UnixFS('raw', Buffer.from('bananas'))
const marshalled = data.marshal()
const unmarshalled = UnixFS.unmarshal(marshalled)
expect(data.type).to.equal(unmarshalled.type)
expect(data.data).to.deep.equal(unmarshalled.data)
expect(data.blockSizes).to.deep.equal(unmarshalled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshalled.fileSize())
const marshaled = data.marshal()
const unmarshaled = UnixFS.unmarshal(marshaled)
expect(data.type).to.equal(unmarshaled.type)
expect(data.data).to.deep.equal(unmarshaled.data)
expect(data.blockSizes).to.deep.equal(unmarshaled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshaled.fileSize())
})

it('directory', () => {
const data = new UnixFS('directory')
const marshalled = data.marshal()
const unmarshalled = UnixFS.unmarshal(marshalled)
expect(data.type).to.equal(unmarshalled.type)
expect(data.data).to.deep.equal(unmarshalled.data)
expect(data.blockSizes).to.deep.equal(unmarshalled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshalled.fileSize())
const marshaled = data.marshal()
const unmarshaled = UnixFS.unmarshal(marshaled)
expect(data.type).to.equal(unmarshaled.type)
expect(data.data).to.deep.equal(unmarshaled.data)
expect(data.blockSizes).to.deep.equal(unmarshaled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshaled.fileSize())
})

it('hamt-sharded-directory', () => {
const data = new UnixFS('hamt-sharded-directory')
const marshalled = data.marshal()
const unmarshalled = UnixFS.unmarshal(marshalled)
expect(data.type).to.equal(unmarshalled.type)
expect(data.data).to.deep.equal(unmarshalled.data)
expect(data.blockSizes).to.deep.equal(unmarshalled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshalled.fileSize())
const marshaled = data.marshal()
const unmarshaled = UnixFS.unmarshal(marshaled)
expect(data.type).to.equal(unmarshaled.type)
expect(data.data).to.deep.equal(unmarshaled.data)
expect(data.blockSizes).to.deep.equal(unmarshaled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshaled.fileSize())
})

it('file', () => {
const data = new UnixFS('file', Buffer.from('batata'))
const marshalled = data.marshal()
const unmarshalled = UnixFS.unmarshal(marshalled)
expect(data.type).to.equal(unmarshalled.type)
expect(data.data).to.deep.equal(unmarshalled.data)
expect(data.blockSizes).to.deep.equal(unmarshalled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshalled.fileSize())
const marshaled = data.marshal()
const unmarshaled = UnixFS.unmarshal(marshaled)
expect(data.type).to.equal(unmarshaled.type)
expect(data.data).to.deep.equal(unmarshaled.data)
expect(data.blockSizes).to.deep.equal(unmarshaled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshaled.fileSize())
})

it('file add blocksize', () => {
const data = new UnixFS('file')
data.addBlockSize(256)
const marshalled = data.marshal()
const unmarshalled = UnixFS.unmarshal(marshalled)
expect(data.type).to.equal(unmarshalled.type)
expect(data.data).to.deep.equal(unmarshalled.data)
expect(data.blockSizes).to.deep.equal(unmarshalled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshalled.fileSize())
const marshaled = data.marshal()
const unmarshaled = UnixFS.unmarshal(marshaled)
expect(data.type).to.equal(unmarshaled.type)
expect(data.data).to.deep.equal(unmarshaled.data)
expect(data.blockSizes).to.deep.equal(unmarshaled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshaled.fileSize())
})

it('file add and remove blocksize', () => {
const data = new UnixFS('file')
data.addBlockSize(256)
const marshalled = data.marshal()
const unmarshalled = UnixFS.unmarshal(marshalled)
expect(data.type).to.equal(unmarshalled.type)
expect(data.data).to.deep.equal(unmarshalled.data)
expect(data.blockSizes).to.deep.equal(unmarshalled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshalled.fileSize())
unmarshalled.removeBlockSize(0)
expect(data.blockSizes).to.not.deep.equal(unmarshalled.blockSizes)
const marshaled = data.marshal()
const unmarshaled = UnixFS.unmarshal(marshaled)
expect(data.type).to.equal(unmarshaled.type)
expect(data.data).to.deep.equal(unmarshaled.data)
expect(data.blockSizes).to.deep.equal(unmarshaled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshaled.fileSize())
unmarshaled.removeBlockSize(0)
expect(data.blockSizes).to.not.deep.equal(unmarshaled.blockSizes)
})

it('default mode for files', () => {
it('mode', () => {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there any way I can convince you to make the spec follow golang's fileMode() specification: golang/go#25422 (comment), or alternatively the less-rigorously defined POSIX st_mode struct?

Having already paid for transporting a 32bit value, it seems so odd to not populate the rest of the bits if we have them. In js-ipfs you can of course simply mask them off, the point is for the spec to allow it...

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there anything in the spec that prevents us filling the higher bits with subsequent PRs?

From my understanding all we've done so far is specify the equivalent of the file mode component of st_mode but the datatype can fit all of st_mode so there's nothing to stop us going further in the future.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@achingbrain yes you are correct. I crossed wires with not explicitly masking off future bits for when we do add them, I left a comment to the effect above

const mode = parseInt('0555', 8)
const data = new UnixFS('file')
expect(data.mode).to.equal(parseInt('0644', 8))
const marshalled = data.marshal()
const unmarshalled = UnixFS.unmarshal(marshalled)
expect(unmarshalled.mode).to.equal(parseInt('0644', 8))
})
data.mode = mode

it('default mode for directories', () => {
const data = new UnixFS('directory')
expect(data.mode).to.equal(parseInt('0755', 8))
const marshalled = data.marshal()
const unmarshalled = UnixFS.unmarshal(marshalled)
expect(unmarshalled.mode).to.equal(parseInt('0755', 8))
expect(UnixFS.unmarshal(data.marshal())).to.have.property('mode', mode)
})

it('default mode for hamt-sharded-directories', () => {
const data = new UnixFS('hamt-sharded-directory')
expect(data.mode).to.equal(parseInt('0755', 8))
const marshalled = data.marshal()
const unmarshalled = UnixFS.unmarshal(marshalled)
expect(unmarshalled.mode).to.equal(parseInt('0755', 8))
it('removes mode', () => {
const mode = parseInt('0555', 8)
const data = new UnixFS('file')
data.mode = mode

const unmarshaled = UnixFS.unmarshal(data.marshal())
expect(unmarshaled).to.have.property('mode', mode)

delete unmarshaled.mode

expect(UnixFS.unmarshal(unmarshaled.marshal())).to.not.have.property('mode')
})

it('mode', () => {
const mode = parseInt('0555', 8)
it('sets mode to 0', () => {
const mode = 0
const data = new UnixFS('file')
data.mode = mode
const marshalled = data.marshal()
const unmarshalled = UnixFS.unmarshal(marshalled)
expect(unmarshalled.mode).to.equal(mode)

expect(UnixFS.unmarshal(data.marshal())).to.have.property('mode', mode)
})

it('mtime', () => {
const mtime = parseInt(Date.now() / 1000)
const mtime = new Date()
const data = new UnixFS('file')
data.mtime = mtime
const marshalled = data.marshal()
const unmarshalled = UnixFS.unmarshal(marshalled)
expect(unmarshalled.mtime).to.equal(mtime)
const marshaled = data.marshal()
const unmarshaled = UnixFS.unmarshal(marshaled)
expect(unmarshaled.mtime).to.deep.equal(new Date(Math.round(mtime.getTime() / 1000) * 1000))
})

it('removes mtime', () => {
const mtime = new Date()
const data = new UnixFS('file')
data.mtime = mtime

const unmarshaled = UnixFS.unmarshal(data.marshal())
expect(unmarshaled).to.have.deep.property('mtime', new Date(Math.round(mtime.getTime() / 1000) * 1000))

delete unmarshaled.mtime

expect(UnixFS.unmarshal(unmarshaled.marshal())).to.not.have.property('mtime')
})

it('sets mtime to 0', () => {
const mtime = new Date(0)
const data = new UnixFS('file')
data.mtime = mtime

expect(UnixFS.unmarshal(data.marshal())).to.have.deep.property('mtime', new Date(Math.round(mtime.getTime() / 1000) * 1000))
})

// figuring out what is this metadata for https://github.com/ipfs/js-ipfs-data-importing/issues/3#issuecomment-182336526
it.skip('metadata', () => {})

it('symlink', () => {
const data = new UnixFS('symlink')
const marshalled = data.marshal()
const unmarshalled = UnixFS.unmarshal(marshalled)
expect(data.type).to.equal(unmarshalled.type)
expect(data.data).to.deep.equal(unmarshalled.data)
expect(data.blockSizes).to.deep.equal(unmarshalled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshalled.fileSize())
const marshaled = data.marshal()
const unmarshaled = UnixFS.unmarshal(marshaled)
expect(data.type).to.equal(unmarshaled.type)
expect(data.data).to.deep.equal(unmarshaled.data)
expect(data.blockSizes).to.deep.equal(unmarshaled.blockSizes)
expect(data.fileSize()).to.deep.equal(unmarshaled.fileSize())
})
it('wrong type', (done) => {
let data
Expand All @@ -146,42 +163,42 @@ describe('unixfs-format', () => {

describe('interop', () => {
it('raw', () => {
const unmarshalled = UnixFS.unmarshal(raw)
expect(unmarshalled.data).to.eql(Buffer.from('Hello UnixFS\n'))
expect(unmarshalled.type).to.equal('file')
expect(unmarshalled.marshal()).to.deep.equal(raw)
const unmarshaled = UnixFS.unmarshal(raw)
expect(unmarshaled.data).to.eql(Buffer.from('Hello UnixFS\n'))
expect(unmarshaled.type).to.equal('file')
expect(unmarshaled.marshal()).to.deep.equal(raw)
})

it('directory', () => {
const unmarshalled = UnixFS.unmarshal(directory)
expect(unmarshalled.data).to.deep.equal(undefined)
expect(unmarshalled.type).to.equal('directory')
expect(unmarshalled.marshal()).to.deep.equal(directory)
const unmarshaled = UnixFS.unmarshal(directory)
expect(unmarshaled.data).to.deep.equal(undefined)
expect(unmarshaled.type).to.equal('directory')
expect(unmarshaled.marshal()).to.deep.equal(directory)
})

it('file', () => {
const unmarshalled = UnixFS.unmarshal(file)
expect(unmarshalled.data).to.deep.equal(Buffer.from('Hello UnixFS\n'))
expect(unmarshalled.type).to.equal('file')
expect(unmarshalled.marshal()).to.deep.equal(file)
const unmarshaled = UnixFS.unmarshal(file)
expect(unmarshaled.data).to.deep.equal(Buffer.from('Hello UnixFS\n'))
expect(unmarshaled.type).to.equal('file')
expect(unmarshaled.marshal()).to.deep.equal(file)
})

it.skip('metadata', () => {
})

it('symlink', () => {
const unmarshalled = UnixFS.unmarshal(symlink)
expect(unmarshalled.data).to.deep.equal(Buffer.from('file.txt'))
expect(unmarshalled.type).to.equal('symlink')
const unmarshaled = UnixFS.unmarshal(symlink)
expect(unmarshaled.data).to.deep.equal(Buffer.from('file.txt'))
expect(unmarshaled.type).to.equal('symlink')
// TODO: waiting on https://github.com/ipfs/js-ipfs-data-importing/issues/3#issuecomment-182440079
// expect(unmarshalled.marshal()).to.deep.equal(symlink)
// expect(unmarshaled.marshal()).to.deep.equal(symlink)
})
})

it('empty', () => {
const data = new UnixFS('file')
const marshalled = data.marshal()
const marshaled = data.marshal()

expect(marshalled).to.deep.equal(Buffer.from([0x08, 0x02, 0x18, 0x00]))
expect(marshaled).to.deep.equal(Buffer.from([0x08, 0x02, 0x18, 0x00]))
})
})