@@ -32,9 +32,15 @@ const CACHE_CONTROL_README: &str = "public,max-age=604800";
32
32
33
33
type StdPath = std:: path:: Path ;
34
34
35
+ #[ derive( Debug ) ]
36
+ pub struct StorageConfig {
37
+ backend : StorageBackend ,
38
+ pub cdn_prefix : Option < String > ,
39
+ }
40
+
35
41
#[ derive( Debug ) ]
36
42
#[ allow( clippy:: large_enum_variant) ]
37
- pub enum StorageConfig {
43
+ pub enum StorageBackend {
38
44
S3 { default : S3Config , index : S3Config } ,
39
45
LocalFileSystem { path : PathBuf } ,
40
46
InMemory ,
@@ -46,10 +52,16 @@ pub struct S3Config {
46
52
region : Option < String > ,
47
53
access_key : String ,
48
54
secret_key : SecretString ,
49
- cdn_prefix : Option < String > ,
50
55
}
51
56
52
57
impl StorageConfig {
58
+ pub fn in_memory ( ) -> Self {
59
+ Self {
60
+ backend : StorageBackend :: InMemory ,
61
+ cdn_prefix : None ,
62
+ }
63
+ }
64
+
53
65
pub fn from_environment ( ) -> Self {
54
66
if let Ok ( bucket) = dotenvy:: var ( "S3_BUCKET" ) {
55
67
let region = dotenvy:: var ( "S3_REGION" ) . ok ( ) ;
@@ -66,18 +78,21 @@ impl StorageConfig {
66
78
region,
67
79
access_key : access_key. clone ( ) ,
68
80
secret_key : secret_key. clone ( ) ,
69
- cdn_prefix,
70
81
} ;
71
82
72
83
let index = S3Config {
73
84
bucket : index_bucket,
74
85
region : index_region,
75
86
access_key,
76
87
secret_key,
77
- cdn_prefix : None ,
78
88
} ;
79
89
80
- return Self :: S3 { default, index } ;
90
+ let backend = StorageBackend :: S3 { default, index } ;
91
+
92
+ return Self {
93
+ backend,
94
+ cdn_prefix,
95
+ } ;
81
96
}
82
97
83
98
let current_dir = std:: env:: current_dir ( )
@@ -86,16 +101,22 @@ impl StorageConfig {
86
101
87
102
let path = current_dir. join ( "local_uploads" ) ;
88
103
89
- Self :: LocalFileSystem { path }
104
+ let backend = StorageBackend :: LocalFileSystem { path } ;
105
+
106
+ Self {
107
+ backend,
108
+ cdn_prefix : None ,
109
+ }
90
110
}
91
111
}
92
112
93
113
pub struct Storage {
114
+ cdn_prefix : Option < String > ,
115
+
94
116
store : Box < dyn ObjectStore > ,
95
117
crate_upload_store : Box < dyn ObjectStore > ,
96
118
readme_upload_store : Box < dyn ObjectStore > ,
97
119
db_dump_upload_store : Box < dyn ObjectStore > ,
98
- cdn_prefix : String ,
99
120
100
121
index_store : Box < dyn ObjectStore > ,
101
122
index_upload_store : Box < dyn ObjectStore > ,
@@ -107,8 +128,10 @@ impl Storage {
107
128
}
108
129
109
130
pub fn from_config ( config : & StorageConfig ) -> Self {
110
- match config {
111
- StorageConfig :: S3 { default, index } => {
131
+ let cdn_prefix = config. cdn_prefix . clone ( ) ;
132
+
133
+ match & config. backend {
134
+ StorageBackend :: S3 { default, index } => {
112
135
let options = ClientOptions :: default ( ) ;
113
136
let store = build_s3 ( default, options) ;
114
137
@@ -122,20 +145,16 @@ impl Storage {
122
145
ClientOptions :: default ( ) . with_default_content_type ( CONTENT_TYPE_DB_DUMP ) ;
123
146
let db_dump_upload_store = build_s3 ( default, options) ;
124
147
125
- let cdn_prefix = match default. cdn_prefix . as_ref ( ) {
126
- None => panic ! ( "Missing S3_CDN environment variable" ) ,
127
- Some ( cdn_prefix) if !cdn_prefix. starts_with ( "https://" ) => {
128
- format ! ( "https://{cdn_prefix}" )
129
- }
130
- Some ( cdn_prefix) => cdn_prefix. clone ( ) ,
131
- } ;
132
-
133
148
let options = ClientOptions :: default ( ) ;
134
149
let index_store = build_s3 ( index, options) ;
135
150
136
151
let options = client_options ( CONTENT_TYPE_INDEX , CACHE_CONTROL_INDEX ) ;
137
152
let index_upload_store = build_s3 ( index, options) ;
138
153
154
+ if cdn_prefix. is_none ( ) {
155
+ panic ! ( "Missing S3_CDN environment variable" ) ;
156
+ }
157
+
139
158
Self {
140
159
store : Box :: new ( store) ,
141
160
crate_upload_store : Box :: new ( crate_upload_store) ,
@@ -147,7 +166,7 @@ impl Storage {
147
166
}
148
167
}
149
168
150
- StorageConfig :: LocalFileSystem { path } => {
169
+ StorageBackend :: LocalFileSystem { path } => {
151
170
warn ! ( ?path, "Using local file system for file storage" ) ;
152
171
153
172
let index_path = path. join ( "index" ) ;
@@ -172,13 +191,13 @@ impl Storage {
172
191
crate_upload_store : Box :: new ( store. clone ( ) ) ,
173
192
readme_upload_store : Box :: new ( store. clone ( ) ) ,
174
193
db_dump_upload_store : Box :: new ( store) ,
175
- cdn_prefix : "/" . into ( ) ,
194
+ cdn_prefix,
176
195
index_store : Box :: new ( index_store. clone ( ) ) ,
177
196
index_upload_store : Box :: new ( index_store) ,
178
197
}
179
198
}
180
199
181
- StorageConfig :: InMemory => {
200
+ StorageBackend :: InMemory => {
182
201
warn ! ( "Using in-memory file storage" ) ;
183
202
let store = ArcStore :: new ( InMemory :: new ( ) ) ;
184
203
@@ -187,7 +206,7 @@ impl Storage {
187
206
crate_upload_store : Box :: new ( store. clone ( ) ) ,
188
207
readme_upload_store : Box :: new ( store. clone ( ) ) ,
189
208
db_dump_upload_store : Box :: new ( store. clone ( ) ) ,
190
- cdn_prefix : "/" . into ( ) ,
209
+ cdn_prefix,
191
210
index_store : Box :: new ( PrefixStore :: new ( store. clone ( ) , "index" ) ) ,
192
211
index_upload_store : Box :: new ( PrefixStore :: new ( store, "index" ) ) ,
193
212
}
@@ -199,14 +218,14 @@ impl Storage {
199
218
///
200
219
/// The function doesn't check for the existence of the file.
201
220
pub fn crate_location ( & self , name : & str , version : & str ) -> String {
202
- format ! ( "{}{}" , self . cdn_prefix, crate_file_path( name, version) ) . replace ( '+' , "%2B" )
221
+ apply_cdn_prefix ( & self . cdn_prefix , & crate_file_path ( name, version) ) . replace ( '+' , "%2B" )
203
222
}
204
223
205
224
/// Returns the URL of an uploaded crate's version readme.
206
225
///
207
226
/// The function doesn't check for the existence of the file.
208
227
pub fn readme_location ( & self , name : & str , version : & str ) -> String {
209
- format ! ( "{}{}" , self . cdn_prefix, readme_path( name, version) ) . replace ( '+' , "%2B" )
228
+ apply_cdn_prefix ( & self . cdn_prefix , & readme_path ( name, version) ) . replace ( '+' , "%2B" )
210
229
}
211
230
212
231
#[ instrument( skip( self ) ) ]
@@ -326,14 +345,24 @@ fn readme_path(name: &str, version: &str) -> Path {
326
345
format ! ( "{PREFIX_READMES}/{name}/{name}-{version}.html" ) . into ( )
327
346
}
328
347
348
+ fn apply_cdn_prefix ( cdn_prefix : & Option < String > , path : & Path ) -> String {
349
+ match cdn_prefix {
350
+ Some ( cdn_prefix) if !cdn_prefix. starts_with ( "https://" ) => {
351
+ format ! ( "https://{cdn_prefix}/{path}" )
352
+ }
353
+ Some ( cdn_prefix) => format ! ( "{cdn_prefix}/{path}" ) ,
354
+ None => format ! ( "/{path}" ) ,
355
+ }
356
+ }
357
+
329
358
#[ cfg( test) ]
330
359
mod tests {
331
360
use super :: * ;
332
361
use hyper:: body:: Bytes ;
333
362
use tempfile:: NamedTempFile ;
334
363
335
364
pub async fn prepare ( ) -> Storage {
336
- let storage = Storage :: from_config ( & StorageConfig :: InMemory ) ;
365
+ let storage = Storage :: from_config ( & StorageConfig :: in_memory ( ) ) ;
337
366
338
367
let files_to_create = vec ! [
339
368
"crates/bar/bar-2.0.0.crate" ,
@@ -361,33 +390,62 @@ mod tests {
361
390
362
391
#[ test]
363
392
fn locations ( ) {
364
- let storage = Storage :: from_config ( & StorageConfig :: InMemory ) ;
393
+ let mut config = StorageConfig :: in_memory ( ) ;
394
+ config. cdn_prefix = Some ( "static.crates.io" . to_string ( ) ) ;
395
+
396
+ let storage = Storage :: from_config ( & config) ;
365
397
366
398
let crate_tests = vec ! [
367
- ( "foo" , "1.2.3" , "/crates/foo/foo-1.2.3.crate" ) ,
399
+ ( "foo" , "1.2.3" , "https://static.crates.io /crates/foo/foo-1.2.3.crate" ) ,
368
400
(
369
401
"some-long-crate-name" ,
370
402
"42.0.5-beta.1+foo" ,
371
- "/crates/some-long-crate-name/some-long-crate-name-42.0.5-beta.1%2Bfoo.crate" ,
403
+ "https://static.crates.io /crates/some-long-crate-name/some-long-crate-name-42.0.5-beta.1%2Bfoo.crate" ,
372
404
) ,
373
405
] ;
374
406
for ( name, version, expected) in crate_tests {
375
407
assert_eq ! ( storage. crate_location( name, version) , expected) ;
376
408
}
377
409
378
410
let readme_tests = vec ! [
379
- ( "foo" , "1.2.3" , "/readmes/foo/foo-1.2.3.html" ) ,
411
+ ( "foo" , "1.2.3" , "https://static.crates.io /readmes/foo/foo-1.2.3.html" ) ,
380
412
(
381
413
"some-long-crate-name" ,
382
414
"42.0.5-beta.1+foo" ,
383
- "/readmes/some-long-crate-name/some-long-crate-name-42.0.5-beta.1%2Bfoo.html" ,
415
+ "https://static.crates.io /readmes/some-long-crate-name/some-long-crate-name-42.0.5-beta.1%2Bfoo.html" ,
384
416
) ,
385
417
] ;
386
418
for ( name, version, expected) in readme_tests {
387
419
assert_eq ! ( storage. readme_location( name, version) , expected) ;
388
420
}
389
421
}
390
422
423
+ #[ test]
424
+ fn cdn_prefix ( ) {
425
+ assert_eq ! ( apply_cdn_prefix( & None , & "foo" . into( ) ) , "/foo" ) ;
426
+ assert_eq ! (
427
+ apply_cdn_prefix( & Some ( "static.crates.io" . to_string( ) ) , & "foo" . into( ) ) ,
428
+ "https://static.crates.io/foo"
429
+ ) ;
430
+ assert_eq ! (
431
+ apply_cdn_prefix(
432
+ & Some ( "https://fastly-static.crates.io" . to_string( ) ) ,
433
+ & "foo" . into( )
434
+ ) ,
435
+ "https://fastly-static.crates.io/foo"
436
+ ) ;
437
+
438
+ assert_eq ! (
439
+ apply_cdn_prefix( & Some ( "static.crates.io" . to_string( ) ) , & "/foo/bar" . into( ) ) ,
440
+ "https://static.crates.io/foo/bar"
441
+ ) ;
442
+
443
+ assert_eq ! (
444
+ apply_cdn_prefix( & Some ( "static.crates.io/" . to_string( ) ) , & "/foo/bar" . into( ) ) ,
445
+ "https://static.crates.io//foo/bar"
446
+ ) ;
447
+ }
448
+
391
449
#[ tokio:: test]
392
450
async fn delete_all_crate_files ( ) {
393
451
let storage = prepare ( ) . await ;
@@ -452,7 +510,7 @@ mod tests {
452
510
453
511
#[ tokio:: test]
454
512
async fn upload_crate_file ( ) {
455
- let s = Storage :: from_config ( & StorageConfig :: InMemory ) ;
513
+ let s = Storage :: from_config ( & StorageConfig :: in_memory ( ) ) ;
456
514
457
515
s. upload_crate_file ( "foo" , "1.2.3" , Bytes :: new ( ) )
458
516
. await
@@ -474,7 +532,7 @@ mod tests {
474
532
475
533
#[ tokio:: test]
476
534
async fn upload_readme ( ) {
477
- let s = Storage :: from_config ( & StorageConfig :: InMemory ) ;
535
+ let s = Storage :: from_config ( & StorageConfig :: in_memory ( ) ) ;
478
536
479
537
let bytes = Bytes :: from_static ( b"hello world" ) ;
480
538
s. upload_readme ( "foo" , "1.2.3" , bytes. clone ( ) )
@@ -495,7 +553,7 @@ mod tests {
495
553
496
554
#[ tokio:: test]
497
555
async fn sync_index ( ) {
498
- let s = Storage :: from_config ( & StorageConfig :: InMemory ) ;
556
+ let s = Storage :: from_config ( & StorageConfig :: in_memory ( ) ) ;
499
557
500
558
assert ! ( stored_files( & s. store) . await . is_empty( ) ) ;
501
559
@@ -512,7 +570,7 @@ mod tests {
512
570
513
571
#[ tokio:: test]
514
572
async fn upload_db_dump ( ) {
515
- let s = Storage :: from_config ( & StorageConfig :: InMemory ) ;
573
+ let s = Storage :: from_config ( & StorageConfig :: in_memory ( ) ) ;
516
574
517
575
assert ! ( stored_files( & s. store) . await . is_empty( ) ) ;
518
576
0 commit comments