Skip to content

Commit 81c6d77

Browse files
robinhood-jimrobinhood-jim
committed
add rapid excel tools
1 parent 06947a9 commit 81c6d77

31 files changed

+1880
-25
lines changed

‎common/pom.xml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,13 @@
323323
<groupId>com.github.chrisvest</groupId>
324324
<artifactId>stormpot</artifactId>
325325
<version>2.4.1</version>
326+
<optional>true</optional>
327+
</dependency>
328+
<dependency>
329+
<groupId>com.fasterxml</groupId>
330+
<artifactId>aalto-xml</artifactId>
331+
<version>1.3.3</version>
332+
<optional>true</optional>
326333
</dependency>
327334
</dependencies>
328335
<build>

‎common/src/main/java/com/robin/comm/util/xls/ExcelBaseOper.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
import org.springframework.util.CollectionUtils;
3737

3838
import java.awt.Color;
39-
import java.io.FileInputStream;
4039
import java.io.IOException;
4140
import java.io.InputStream;
4241
import java.nio.file.Files;
@@ -76,7 +75,7 @@ public static Workbook createWorkBook(ExcelSheetProp prop) {
7675
})
7776
.orElseGet(HSSFWorkbook::new);
7877
} else {
79-
wb = Optional.of(prop.isStreamInsert()).map(f -> {
78+
wb = Optional.of(prop.isStreamMode()).map(f -> {
8079
Workbook wb1 = Optional.ofNullable(prop.getTemplateFile()).map(p1 -> {
8180
try {
8281
SXSSFWorkbook wb3 = new SXSSFWorkbook(new XSSFWorkbook(getTemplateInputStream(prop)), prop.getStreamRows());

‎common/src/main/java/com/robin/comm/util/xls/ExcelCellStyleUtil.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import java.util.Map;
1919

20+
import com.robin.core.base.util.DateTimeFormatHolder;
2021
import org.apache.poi.hssf.usermodel.HSSFDataFormat;
2122
import org.apache.poi.ss.usermodel.*;
2223

@@ -30,6 +31,7 @@ private ExcelCellStyleUtil() {
3031

3132
}
3233

34+
3335
public static CellStyle getNoBorderCellType(Workbook wb, String metaType) {
3436
CellStyle cs = wb.createCellStyle();
3537
cs.setAlignment(HorizontalAlignment.CENTER);
@@ -52,7 +54,7 @@ private static void extractMeta(Workbook wb,String metaType, CellStyle cs) {
5254
break;
5355
case Const.META_TYPE_DATE:
5456
case Const.META_TYPE_TIMESTAMP:
55-
cs.setDataFormat(wb.getCreationHelper().createDataFormat().getFormat("yyyy-MM-dd hh:mm:ss"));
57+
cs.setDataFormat(wb.getCreationHelper().createDataFormat().getFormat(DateTimeFormatHolder.getTimestampFormatter().toString()));
5658
break;
5759
case Const.META_TYPE_INTEGER:
5860
cs.setDataFormat(HSSFDataFormat.getBuiltinFormat("0"));

‎common/src/main/java/com/robin/comm/util/xls/ExcelProcessor.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -476,7 +476,7 @@ public static Workbook generateExcelFile(ExcelSheetProp prop, TableConfigProp he
476476
while (iterator.hasNext()) {
477477
Map<String, Object> map = iterator.next();
478478
processSingleLine(map, wb, sheet, row, prop, header, helper, cellMap);
479-
if (prop.isStreamInsert() && (row + 1) % prop.getStreamRows() == 0) {
479+
if (prop.isStreamMode() && (row + 1) % prop.getStreamRows() == 0) {
480480
((SXSSFSheet) sheet).flushRows(prop.getStreamRows());
481481
}
482482
row++;
@@ -688,7 +688,7 @@ private static void fillColumns(Workbook wb, Sheet targetsheet, ExcelSheetProp p
688688
Map<String, CellStyle> cellMap = new HashMap<>();
689689
for (int i = 0; i < list.size(); i++) {
690690
processSingleLine(list.get(i), wb, targetsheet, i, prop, header, helper, cellMap);
691-
if (prop.isStreamInsert() && (i + 1) % prop.getStreamRows() == 0) {
691+
if (prop.isStreamMode() && (i + 1) % prop.getStreamRows() == 0) {
692692
((SXSSFSheet) targetsheet).flushRows(prop.getStreamRows());
693693
}
694694
}
@@ -817,7 +817,7 @@ public static void processSingleLine(Map<String, ?> map, Workbook wb, Sheet targ
817817

818818
private static void autoSizeSheet(ExcelSheetProp prop, Sheet sheet, int count) {
819819
for (int i = 0; i < count; i++) {
820-
if (!prop.isStreamInsert()) {
820+
if (!prop.isStreamMode()) {
821821
sheet.autoSizeColumn(i);
822822
} else {
823823
if (i == 0) {

‎common/src/main/java/com/robin/comm/util/xls/ExcelRsExtractor.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public boolean executeAdditionalOperation(Map<String, Object> map, ResultSetMeta
3838
pos++;
3939
processRows++;
4040
try {
41-
if (prop.isStreamInsert() && (processRows) % prop.getStreamRows() == 0) {
41+
if (prop.isStreamMode() && (processRows) % prop.getStreamRows() == 0) {
4242
((SXSSFSheet) targetSheet).flushRows(prop.getStreamRows());
4343
}
4444

‎common/src/main/java/com/robin/comm/util/xls/ExcelSheetProp.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ public class ExcelSheetProp {
3232
private int startCol=1;
3333
private Integer tableId;
3434
// using SXSSFWorkbook with streamingWrite
35-
private boolean streamInsert=false;
35+
private boolean streamMode =false;
3636
private Integer streamRows=100;
3737
private int sheetNum=0;
3838
private boolean fillHeader=true;

‎common/src/main/java/com/robin/core/fileaccess/iterator/XlsxFileIterator.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,14 @@ public XlsxFileIterator(DataCollectionMeta meta, AbstractFileSystemAccessor acce
5959
public void beforeProcess() {
6060
super.beforeProcess();
6161
try {
62-
factory = XMLInputFactory.newFactory();
62+
factory = new com.fasterxml.aalto.stax.InputFactoryImpl();
6363
factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE);
6464
factory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
6565
opcPackage=OPCPackage.open(instream);
6666
xssfReader=new XSSFReader(opcPackage);
6767
sheetStreams=xssfReader.getSheetsData();
6868
sharedStrings=xssfReader.getSharedStringsTable();
69+
6970
if(sheetStreams.hasNext()){
7071
readStreams=sheetStreams.next();
7172
streamReader=factory.createXMLStreamReader(readStreams,colmeta.getEncode());

‎common/src/main/java/com/robin/core/fileaccess/util/ByteBufferInputStream.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@ public ByteBufferInputStream(ByteBuffer byteBuffer,int count){
1616
byteBuffer.position(0);
1717
this.count=count;
1818
}
19+
public ByteBufferInputStream(ByteBuffer buffer,byte[] bytes){
20+
this.byteBuffer=buffer;
21+
byteBuffer.position(0);
22+
buffer.put(bytes);
23+
this.count=bytes.length;
24+
}
1925
@Override
2026
public int read() throws IOException {
2127
if (byteBuffer.position()>count || byteBuffer.remaining() == 0) {
@@ -83,4 +89,9 @@ public int capacity(){
8389
public void seek(int pos){
8490
byteBuffer.position(pos);
8591
}
92+
93+
@Override
94+
public void close() throws IOException {
95+
byteBuffer.clear();
96+
}
8697
}

‎common/src/main/java/com/robin/core/fileaccess/writer/XlsxFileWriter.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public XlsxFileWriter(DataCollectionMeta colmeta, AbstractFileSystemAccessor acc
8484
public void beginWrite() throws IOException {
8585
super.beginWrite();
8686
try {
87-
XMLOutputFactory factory=XMLOutputFactory.newFactory();
87+
XMLOutputFactory factory=new com.fasterxml.aalto.stax.OutputFactoryImpl();
8888
workbook = new XSSFWorkbook();
8989
Field field = workbook.getClass().getSuperclass().getDeclaredField("pkg");
9090
field.setAccessible(true);
@@ -96,6 +96,7 @@ public void beginWrite() throws IOException {
9696
ExcelCellStyleUtil.getCellStyle(workbook, colmeta.getColumnList().get(i).getColumnType(), cellStyleMap);
9797
}
9898
byteOut=new ByteArrayOutputStream();
99+
99100
if(!multipleSheets) {
100101
workbook.write(byteOut);
101102
workbook.close();

‎common/src/test/java/com/robin/comm/test/TestExcelOperation.java

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,24 +12,19 @@
1212
import com.robin.core.fileaccess.iterator.AbstractResIterator;
1313
import com.robin.core.fileaccess.iterator.TextFileIteratorFactory;
1414
import com.robin.core.fileaccess.meta.DataCollectionMeta;
15-
import com.robin.core.fileaccess.writer.AbstractFileWriter;
1615
import com.robin.core.fileaccess.writer.TextFileWriterFactory;
1716
import com.robin.core.fileaccess.writer.XlsxFileWriter;
1817
import lombok.extern.slf4j.Slf4j;
1918
import org.apache.commons.collections.CollectionUtils;
20-
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
21-
import org.apache.poi.hssf.usermodel.HSSFDataFormat;
2219
import org.apache.poi.openxml4j.opc.*;
2320
import org.apache.poi.ss.usermodel.*;
2421
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
25-
import org.apache.poi.xssf.usermodel.XSSFRelation;
2622
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
2723
import org.junit.Test;
2824
import org.slf4j.Logger;
2925
import org.slf4j.LoggerFactory;
3026
import org.springframework.util.ObjectUtils;
3127

32-
import javax.xml.stream.XMLInputFactory;
3328
import javax.xml.stream.XMLOutputFactory;
3429
import javax.xml.stream.XMLStreamWriter;
3530
import java.io.*;
@@ -56,18 +51,17 @@ public void testReadWithIterator() throws Exception {
5651
builder.addColumn("name", Const.META_TYPE_STRING).addColumn("time", Const.META_TYPE_TIMESTAMP)
5752
.addColumn("intcol", Const.META_TYPE_INTEGER).addColumn("dval", Const.META_TYPE_DOUBLE)
5853
.addColumn("dval2", Const.META_TYPE_DOUBLE).addColumn("diff", Const.META_TYPE_DOUBLE)
59-
.fileFormat(Const.FILEFORMATSTR.XLSX.getValue()).resPath("file:///d:/test.xlsx");
54+
.fileFormat(Const.FILEFORMATSTR.XLSX.getValue()).resPath("file:///d:/test2.xlsx");
6055

6156
LocalFileSystemAccessor accessor = LocalFileSystemAccessor.getInstance();
57+
Long fromTs=System.currentTimeMillis();
6258
try (AbstractFileIterator iterator = (AbstractFileIterator) TextFileIteratorFactory.getProcessIteratorByType(builder.build(), accessor)) {
6359
int pos = 0;
6460
while (iterator.hasNext()) {
6561
pos++;
66-
if (pos % 10000 == 0) {
67-
System.out.println(iterator.next());
68-
}
6962
}
7063
System.out.println(pos);
64+
System.out.println(System.currentTimeMillis()-fromTs);
7165
} catch (IOException ex) {
7266

7367
}
@@ -94,7 +88,7 @@ public void testWriteStAX() throws Exception{
9488
writer.setSheetProp(prop);
9589
writer.beginWrite();
9690
Long startTs = System.currentTimeMillis() - 3600 * 24 * 1000;
97-
for(int j=0;j<5000;j++){
91+
for(int j=0;j<120000;j++){
9892
cachedMap.put("name", StringUtils.generateRandomChar(12));
9993
cachedMap.put("time", String.valueOf(startTs + j * 1000));
10094
cachedMap.put("intcol", String.valueOf(random.nextInt(1000)));
@@ -319,7 +313,7 @@ public void testGenerate() throws Exception {
319313
prop.setStartCol(1);
320314
prop.setStartRow(1);
321315
prop.setSheetName("test");
322-
prop.setStreamInsert(true);
316+
prop.setStreamMode(true);
323317
prop.addColumnProp(new ExcelColumnProp("name", "name", Const.META_TYPE_STRING, false));
324318
prop.addColumnProp(new ExcelColumnProp("time", "time", Const.META_TYPE_TIMESTAMP, false));
325319
prop.addColumnProp(new ExcelColumnProp("intcol", "intcol", Const.META_TYPE_INTEGER, false));
@@ -388,7 +382,7 @@ public void testRead() throws IOException {
388382
prop.setStartCol(1);
389383
prop.setStartRow(1);
390384
prop.setSheetName("test");
391-
prop.setStreamInsert(true);
385+
prop.setStreamMode(true);
392386
prop.addColumnProp(new ExcelColumnProp("name", "name", Const.META_TYPE_STRING, false));
393387
prop.addColumnProp(new ExcelColumnProp("time", "time", Const.META_TYPE_TIMESTAMP, false));
394388
prop.addColumnProp(new ExcelColumnProp("intcol", "intcol", Const.META_TYPE_INTEGER, false));
@@ -419,7 +413,7 @@ public void testGenWithQuery() {
419413
prop.addColumnProp(new ExcelColumnProp("model", "model", Const.META_TYPE_STRING, false));
420414
prop.addColumnProp(new ExcelColumnProp("engine", "engine", Const.META_TYPE_STRING, false));
421415
prop.addColumnProp(new ExcelColumnProp("img", "img", Const.META_TYPE_STRING, false));
422-
prop.setStreamInsert(true);
416+
prop.setStreamMode(true);
423417
prop.setStreamRows(3000);
424418
TableConfigProp header = new TableConfigProp();
425419
header.setContainrow(1);

‎core/src/main/java/com/robin/core/base/dao/JdbcDao.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -520,7 +520,7 @@ public <T extends BaseObject, P extends Serializable> P createVO(T obj, Class<P>
520520
}
521521
FieldContent generateColumn;
522522
//pk model insert
523-
if (insertSegment.isHasPrimaryKey()) {
523+
if (insertSegment.isHasincrementPk() || insertSegment.isHasSequencePk()) {
524524
PreparedStatementCreatorFactory factory = new PreparedStatementCreatorFactory(insertSql, insertSegment.getParamTypes());
525525
KeyHolder keyHolder = new GeneratedKeyHolder();
526526
if (insertSegment.isHasSequencePk()) {
@@ -555,7 +555,7 @@ public <T extends BaseObject, P extends Serializable> P createVO(T obj, Class<P>
555555

556556
}
557557
} else {
558-
//no pk model insert
558+
//no pk model insert or assign value pk
559559
if (!insertSegment.isContainlob()) {
560560
executeUpdate(insertSql, fields, obj);
561561
} else {

‎core/src/main/java/com/robin/core/base/util/FileUtils.java

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import com.robin.core.fileaccess.meta.DataCollectionMeta;
66
import lombok.Data;
77
import lombok.extern.slf4j.Slf4j;
8+
import org.apache.commons.io.FileExistsException;
89
import org.springframework.util.Assert;
910
import org.springframework.util.MimeTypeUtils;
1011
import org.springframework.util.ObjectUtils;
@@ -19,6 +20,7 @@
1920
import java.nio.file.attribute.PosixFilePermission;
2021
import java.nio.file.attribute.UserPrincipalLookupService;
2122
import java.util.*;
23+
import java.util.stream.Collectors;
2224

2325
@Slf4j
2426
public class FileUtils {
@@ -152,6 +154,38 @@ public static boolean setWithGroupAndUser(File file,String group,String user){
152154
}
153155
return true;
154156
}
157+
public static boolean mkDirReclusive(String path) throws FileExistsException{
158+
int pos=path.contains(File.separator)?path.lastIndexOf(File.separator):path.lastIndexOf("/");
159+
Assert.isTrue(pos>0,"not a valid filePath");
160+
String processPath=path.substring(0,pos);
161+
if(processPath.startsWith("file:///")){
162+
processPath=processPath.substring(8);
163+
}
164+
List<String> pathParts=processPath.contains(File.separator)?Lists.newArrayList(processPath.split(File.separator)):Lists.newArrayList(processPath.split("/"));
165+
if(pathParts.size()==1){
166+
return true;
167+
}
168+
StringBuilder builder=new StringBuilder();
169+
170+
builder.append(pathParts.get(0)).append(File.separator);
171+
for(int i=1;i<pathParts.size();i++){
172+
builder.append(pathParts.get(i));
173+
mkdir(builder.toString());
174+
builder.append(File.separator);
175+
}
176+
return true;
177+
178+
}
179+
public static boolean mkdir(String path) throws FileExistsException {
180+
File file=new File(path);
181+
if(file.isDirectory()){
182+
return true;
183+
}else if(!file.exists()){
184+
return file.mkdir();
185+
}else{
186+
throw new FileExistsException("path already exists as file");
187+
}
188+
}
155189
public static String getWorkingPath(DataCollectionMeta meta){
156190
return !ObjectUtils.isEmpty(meta.getResourceCfgMap().get(ResourceConst.WORKINGPATHPARAM))
157191
? meta.getResourceCfgMap().get(ResourceConst.WORKINGPATHPARAM).toString()

0 commit comments

Comments
 (0)