git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
6
function pullOnce(data) {
7
  var ended
8
  return function (abort, cb) {
9
    if (ended || (ended = abort)) return cb(ended)
10
    ended = true
11
    cb(null, data)
12
  }
13
}
14
15
exports.name = 'npm-registry'
16
exports.version = '1.0.0'
17
exports.manifest = {
18
  getAddress: 'async'
19
}
20
exports.init = function (sbot, config) {
21
  var port = config.npm ? config.npm.port : 8043
22
  var host = config.npm && config.npm.host || 'localhost'
23
  var registryAddress
24
  var getAddressCbs = []
25
26
  var server = http.createServer(exports.respond(sbot, config))
27
  server.listen(port, host, function () {
28
    registryAddress = 'http://' + host + ':' + this.address().port + '/'
29
    if (config.npm && config.npm.autoAuth !== false) login()
30
    console.log('[npm-registry] Listening on ' + registryAddress)
31
    while (getAddressCbs.length) getAddressCbs.shift()(null, registryAddress)
32
  })
33
  sbot.on('close', function () {
34
    server.close()
35
  })
36
37
  function login() {
38
    var filename = path.join(os.homedir(), '.npmrc')
39
    var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
40
    var lines = fs.readFileSync(filename, 'utf8').split('\n')
41
    if (lines.indexOf(tokenLine) === -1) {
42
      fs.appendFileSync(filename, (lines.pop() ? '' : '\n') + tokenLine)
43
    }
44
  }
45
46
  return {
47
    getAddress: function (cb) {
48
      if (registryAddress) cb(null, registryAddress)
49
      else getAddressCbs.push(cb)
50
    }
51
  }
52
}
53
54
exports.respond = function (sbot, config) {
55
  var reg = new SsbNpmRegistryServer(sbot, config)
56
  return function (req, res) {
57
    new Req(reg, req, res).serve()
58
  }
59
}
60
61
function publishMsg(sbot, value, cb) {
62
  var gotExpectedPrevious = false
63
  sbot.publish(value, function next(err, msg) {
64
    if (err && /^expected previous:/.test(err.message)) {
65
      // retry once on this error
66
      if (gotExpectedPrevious) return cb(err)
67
      gotExpectedPrevious = true
68
      return sbot.publish(value, next)
69
    }
70
    cb(err, msg)
71
  })
72
}
73
74
function publishMentions(sbot, mentions, cb) {
75
  // console.error("publishing %s mentions", mentions.length)
76
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
77
  publishMsg(sbot, {
78
    type: 'npm-packages',
79
    mentions: mentions,
80
  }, cb)
81
}
82
83
exports.publishPkgMentions = function (sbot, mentions, cb) {
84
  // try to fit the mentions into as few messages as possible,
85
  // while fitting under the message size limit.
86
  var msgs = []
87
  ;(function next(i, chunks) {
88
    if (i >= mentions.length) return cb(null, msgs)
89
    var chunkLen = Math.ceil(mentions.length / chunks)
90
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
91
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
92
      if (err && msgs.length) return onPartialPublish(err)
93
      if (err) return cb(err)
94
      msgs.push(msg)
95
      next(i + chunkLen, chunks)
96
    })
97
  })(0, 1)
98
  function onPartialPublish(err) {
99
    var remaining = mentions.length - i
100
    return cb(new Error('Published messages ' +
101
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
102
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
103
  }
104
}
105
106
function SsbNpmRegistryServer(sbot, config) {
107
  this.sbot = sbot
108
  this.config = config
109
  this.links2 = sbot.links2
110
  if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin')
111
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
112
    + (config.ws && config.ws.port || '8989') + '/blobs/get/'
113
}
114
115
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
116
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
117
118
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
119
  var self = this
120
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
121
  ;(function next(i) {
122
    if (i >= ids.length) return cb()
123
    self.sbot.blobs.push(ids[i], function (err) {
124
      if (err) return cb(err)
125
      next(i+1)
126
    })
127
  })(0)
128
}
129
130
SsbNpmRegistryServer.prototype.blobUrl = function (id) {
131
  return this.blobsPrefix + id
132
}
133
134
SsbNpmRegistryServer.prototype.getMentions = function (name) {
135
  return this.links2.read({
136
    query: [
137
      {$filter: {rel: ['mentions', name]}},
138
      {$filter: {dest: {$prefix: '&'}}},
139
      {$map: {
140
        name: ['rel', 1],
141
        size: ['rel', 2],
142
        link: 'dest',
143
        author: 'source',
144
        ts: 'ts'
145
      }}
146
    ]
147
  })
148
}
149
150
function Req(server, req, res) {
151
  this.server = server
152
  this.req = req
153
  this.res = res
154
  this.blobsToPush = []
155
}
156
157
Req.prototype.serve = function () {
158
  // console.log(this.req.method, this.req.url)
159
  var pathname = this.req.url.replace(/\?.*/, '')
160
  if (pathname === '/-/whoami') return this.serveWhoami()
161
  if (pathname === '/-/ping') return this.respond(200, true)
162
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
163
  return this.respond(404)
164
}
165
166
Req.prototype.respond = function (status, message) {
167
  this.res.writeHead(status, {'content-type': 'application/json'})
168
  this.res.end(message && JSON.stringify(message))
169
}
170
171
Req.prototype.respondError = function (status, message) {
172
  this.respond(status, {error: message})
173
}
174
175
Req.prototype.serveWhoami = function () {
176
  var self = this
177
  self.server.sbot.whoami(function (err, feed) {
178
    if (err) return self.respondError(err.stack || err)
179
    self.respond(200, {username: feed.id})
180
  })
181
}
182
183
function decodeName(name) {
184
  var parts = name.replace(/\.tgz$/, '').split(':')
185
  return {
186
    name: parts[1],
187
    version: parts[2],
188
    distTag: parts[3],
189
  }
190
}
191
192
Req.prototype.servePkg = function (pathname) {
193
  var self = this
194
  var parts = pathname.split('/')
195
  var pkgName = parts.shift()
196
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
197
  if (parts.length > 0) return this.respondError(404)
198
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
199
  var obj = {
200
    _id: pkgName,
201
    name: pkgName,
202
    'dist-tags': {},
203
    versions: {}
204
  }
205
  var oldest, newest
206
  var getMention = self.server.getMentions({$prefix: 'npm:' + pkgName + ':'})
207
  getMention(null, function next(err, mention) {
208
    if (err === true) return self.respond(200, obj)
209
    if (err) return self.respondError(500, err.stack || err)
210
    var data = decodeName(mention.name)
211
    if (!data.version) return
212
    var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(mention.link)
213
    if (!m) return
214
    if (data.distTag) obj['dist-tags'][data.distTag] = data.version
215
    obj.versions[data.version] = {
216
      author: {
217
        url: mention.author
218
      },
219
      dist: {
220
        integrity: m[2] + '-' + m[1],
221
        tarball: self.server.blobUrl(mention.link)
222
      }
223
    }
224
    getMention(null, next)
225
  })
226
}
227
228
Req.prototype.publishPkg = function (pkgName) {
229
  var chunks = []
230
  var self = this
231
  self.req.on('data', function (data) {
232
    chunks.push(data)
233
  })
234
  self.req.on('end', function () {
235
    var data
236
    try {
237
      data = JSON.parse(Buffer.concat(chunks))
238
    } catch(e) {
239
      return self.respondError(400, e.stack)
240
    }
241
    return self.publishPkg2(pkgName, data || {})
242
  })
243
}
244
245
Req.prototype.publishPkg2 = function (name, data) {
246
  var self = this
247
  if (data.users) console.trace('[npm-registry users property is not supported')
248
  var attachments = data._attachments || {}
249
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
250
  var waiting = 0
251
  Object.keys(attachments).forEach(function (filename) {
252
    waiting++
253
    var tarball = new Buffer(attachments[filename].data, 'base64')
254
    var length = attachments[filename].length
255
    if (length && length !== tarball.length) return self.respondError(400,
256
      'Length mismatch for attachment \'' + filename + '\'')
257
    self.server.sbot.blobs.add(function (err, id) {
258
      if (err) return self.respondError(500,
259
        'Adding attachment \'' + filename + '\' as blob failed')
260
      self.blobsToPush.push(id)
261
      links[filename] = {link: id, size: tarball.length}
262
      if (!--waiting) next()
263
    })(pullOnce(tarball))
264
  })
265
  function next() {
266
    try {
267
      self.publishPkg3(name, data, links)
268
    } catch(e) {
269
      self.respondError(500, e.stack || e)
270
    }
271
  }
272
}
273
274
Req.prototype.publishPkg3 = function (name, data, links) {
275
  var self = this
276
  var versions = data.versions || {}
277
  var linksByVersion = {/* <version>: link */}
278
279
  // associate tarball blobs with versions
280
  for (var version in versions) {
281
    var pkg = versions[version]
282
    if (!pkg) return self.respondError(400, 'Bad package object')
283
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
284
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
285
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
286
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
287
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
288
    var filename = m[1]
289
    var link = links[filename]
290
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
291
    // TODO?: try to find missing tarball mentioned in other messages
292
    if (pkg.version && pkg.version !== version)
293
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
294
    linksByVersion[version] = link
295
    link.version = version
296
  }
297
298
  // associate blobs with dist-tags
299
  var tags = data['dist-tags'] || {}
300
  for (var tag in tags) {
301
    var version = tags[tag]
302
    var link = linksByVersion[version]
303
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
304
    // TODO?: support setting dist-tag without version,
305
    // by looking up a tarball blob for the version
306
    link.tag = tag
307
  }
308
309
  // compute blob links to publish
310
  var mentions = []
311
  for (var filename in links) {
312
    var link = links[filename] || {}
313
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
314
    mentions.push({
315
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
316
      link: link.link,
317
      size: link.size,
318
    })
319
  }
320
  return self.publishPkgs(mentions)
321
}
322
323
Req.prototype.publishPkgs = function (mentions) {
324
  var self = this
325
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
326
    if (err) self.respondError(500, err.stack || err)
327
    self.server.pushBlobs(self.blobsToPush, function (err) {
328
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
329
      self.respond(201)
330
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
331
    })
332
  })
333
}
334