@@ -18,8 +18,14 @@ export type Options = {
18
18
stream : SyncStream ;
19
19
deterministicCQId : boolean ;
20
20
concurrency : number ;
21
+ strategy ?: Strategy ;
21
22
} ;
22
23
24
+ export enum Strategy {
25
+ dfs = 'dfs' ,
26
+ roundRobin = 'round-robin' ,
27
+ }
28
+
23
29
class TableResolverStream extends Duplex {
24
30
queue : unknown [ ] = [ ] ;
25
31
@@ -100,19 +106,81 @@ const resolveTable = async (
100
106
101
107
syncStream . write ( new SyncResponse ( { insert : new Insert ( { record : encodeResource ( resource ) } ) } ) ) ;
102
108
103
- await Promise . all ( table . relations . map ( ( child ) => resolveTable ( logger , client , child , resource , syncStream ) ) ) ;
109
+ await pMap ( table . relations , ( child ) => resolveTable ( logger , client , child , resource , syncStream ) ) ;
104
110
}
105
111
} ;
106
112
107
- export const sync = async ( { logger, client, tables, stream : syncStream , concurrency } : Options ) => {
113
+ const syncDfs = async ( { logger, client, tables, stream : syncStream , concurrency } : Omit < Options , 'strategy' > ) => {
114
+ const tableClients = tables . flatMap ( ( table ) => {
115
+ const clients = table . multiplexer ( client ) ;
116
+ return clients . map ( ( client ) => ( { table, client } ) ) ;
117
+ } ) ;
118
+
119
+ await pMap ( tableClients , ( { table, client } ) => resolveTable ( logger , client , table , null , syncStream ) , {
120
+ concurrency,
121
+ } ) ;
122
+ } ;
123
+
124
+ export const getRoundRobinTableClients = ( tables : Table [ ] , client : ClientMeta ) => {
125
+ let tablesWithClients = tables
126
+ . map ( ( table ) => ( { table, clients : table . multiplexer ( client ) } ) )
127
+ . filter ( ( { clients } ) => clients . length > 0 ) ;
128
+
129
+ const tableClients : { table : Table ; client : ClientMeta } [ ] = [ ] ;
130
+ while ( tablesWithClients . length > 0 ) {
131
+ for ( const { table, clients } of tablesWithClients ) {
132
+ tableClients . push ( { table, client : clients . shift ( ) as ClientMeta } ) ;
133
+ }
134
+ tablesWithClients = tablesWithClients . filter ( ( { clients } ) => clients . length > 0 ) ;
135
+ }
136
+
137
+ return tableClients ;
138
+ } ;
139
+
140
+ const syncRoundRobin = async ( {
141
+ logger,
142
+ client,
143
+ tables,
144
+ stream : syncStream ,
145
+ concurrency,
146
+ } : Omit < Options , 'strategy' > ) => {
147
+ const tableClients = getRoundRobinTableClients ( tables , client ) ;
148
+ await pMap ( tableClients , ( { table, client } ) => resolveTable ( logger , client , table , null , syncStream ) , {
149
+ concurrency,
150
+ } ) ;
151
+ } ;
152
+
153
+ export const sync = async ( {
154
+ logger,
155
+ client,
156
+ tables,
157
+ stream,
158
+ concurrency,
159
+ strategy = Strategy . dfs ,
160
+ deterministicCQId,
161
+ } : Options ) => {
108
162
for ( const table of tables ) {
109
163
logger . info ( `sending migrate message for table ${ table . name } ` ) ;
110
164
// eslint-disable-next-line @typescript-eslint/naming-convention
111
- syncStream . write ( new SyncResponse ( { migrate_table : new MigrateTable ( { table : encodeTable ( table ) } ) } ) ) ;
165
+ stream . write ( new SyncResponse ( { migrate_table : new MigrateTable ( { table : encodeTable ( table ) } ) } ) ) ;
112
166
}
113
167
114
- await pMap ( tables , ( table ) => resolveTable ( logger , client , table , null , syncStream ) , { concurrency } ) ;
168
+ switch ( strategy ) {
169
+ case Strategy . dfs : {
170
+ logger . debug ( `using dfs strategy` ) ;
171
+ await syncDfs ( { logger, client, tables, stream, concurrency, deterministicCQId } ) ;
172
+ break ;
173
+ }
174
+ case Strategy . roundRobin : {
175
+ logger . debug ( `using round-robin strategy` ) ;
176
+ await syncRoundRobin ( { logger, client, tables, stream, concurrency, deterministicCQId } ) ;
177
+ break ;
178
+ }
179
+ default : {
180
+ throw new Error ( `unknown strategy ${ strategy } ` ) ;
181
+ }
182
+ }
115
183
116
- syncStream . end ( ) ;
184
+ stream . end ( ) ;
117
185
return await Promise . resolve ( ) ;
118
186
} ;
0 commit comments