Skip to content

Commit 3651fd9

Browse files
php-coderphp-coder
committed
Series import: extract quantity of the stamps.
Fix #781
1 parent 0df664c commit 3651fd9

19 files changed

+134
-9
lines changed

‎src/main/java/ru/mystamps/web/controller/SeriesImportController.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,9 @@ public String showRequestAndImportSeriesForm(
128128
if (parsedData.getIssueYear() != null) {
129129
form.setYear(parsedData.getIssueYear());
130130
}
131+
if (parsedData.getQuantity() != null) {
132+
form.setQuantity(parsedData.getQuantity());
133+
}
131134
}
132135

133136
model.addAttribute("importSeriesForm", form);

‎src/main/java/ru/mystamps/web/controller/event/DownloadingSucceededEventListener.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,8 @@ public void onApplicationEvent(DownloadingSucceeded event) {
9090
info.getCategoryName(),
9191
info.getCountryName(),
9292
info.getImageUrl(),
93-
info.getIssueDate()
93+
info.getIssueDate(),
94+
info.getQuantity()
9495
);
9596
importService.saveParsedData(requestId, data);
9697
}

‎src/main/java/ru/mystamps/web/dao/dto/ParsedDataDto.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,5 @@ public class ParsedDataDto {
2727
private final LinkEntityDto country;
2828
private final String imageUrl;
2929
private final Integer issueYear;
30+
private final Integer quantity;
3031
}

‎src/main/java/ru/mystamps/web/dao/dto/SaveParsedDataDbDto.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,16 @@ public class SaveParsedDataDbDto {
3131
private Integer countryId;
3232
private String imageUrl;
3333
private Integer releaseYear;
34+
private Integer quantity;
3435
private Date createdAt;
3536
private Date updatedAt;
3637

3738
public boolean hasAtLeastOneFieldFilled() {
3839
return categoryId != null
3940
|| countryId != null
4041
|| imageUrl != null
41-
|| releaseYear != null;
42+
|| releaseYear != null
43+
|| quantity != null;
4244
}
4345

4446
}

‎src/main/java/ru/mystamps/web/dao/impl/JdbcSeriesImportDao.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,7 @@ public void addParsedContent(Integer requestId, SaveParsedDataDbDto data) {
211211
params.put("release_year", data.getReleaseYear());
212212
params.put("created_at", data.getCreatedAt());
213213
params.put("updated_at", data.getUpdatedAt());
214+
params.put("quantity", data.getQuantity());
214215

215216
KeyHolder holder = new GeneratedKeyHolder();
216217

‎src/main/java/ru/mystamps/web/dao/impl/RowMappers.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -273,8 +273,9 @@ public static ParsedDataDto forParsedDataDto(ResultSet rs, int i) throws SQLExce
273273

274274
String imageUrl = rs.getString("image_url");
275275
Integer releaseYear = JdbcUtils.getInteger(rs, "release_year");
276+
Integer quantity = JdbcUtils.getInteger(rs, "quantity");
276277

277-
return new ParsedDataDto(category, country, imageUrl, releaseYear);
278+
return new ParsedDataDto(category, country, imageUrl, releaseYear, quantity);
278279
}
279280

280281
public static ImportRequestInfo forImportRequestInfo(ResultSet rs, int i) throws SQLException {

‎src/main/java/ru/mystamps/web/service/SeriesImportServiceImpl.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,11 @@ public void saveParsedData(Integer requestId, RawParsedDataDto data) {
185185
processedData.setReleaseYear(releaseYear);
186186
}
187187

188+
Integer quantity = extractorService.extractQuantity(data.getQuantity());
189+
if (quantity != null) {
190+
processedData.setQuantity(quantity);
191+
}
192+
188193
// IMPORTANT: don't add code that modifies database above this line!
189194
// @todo #684 Series import: add integration test
190195
// for the case when parsed value don't match database

‎src/main/java/ru/mystamps/web/service/SeriesInfoExtractorService.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,5 @@ public interface SeriesInfoExtractorService {
2323
List<Integer> extractCategory(String fragment);
2424
List<Integer> extractCountry(String fragment);
2525
Integer extractReleaseYear(String fragment);
26+
Integer extractQuantity(String fragment);
2627
}

‎src/main/java/ru/mystamps/web/service/SeriesInfoExtractorServiceImpl.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.util.HashSet;
2222
import java.util.List;
2323
import java.util.Set;
24+
import java.util.regex.Matcher;
2425
import java.util.regex.Pattern;
2526

2627
import org.apache.commons.lang3.StringUtils;
@@ -41,6 +42,10 @@ public class SeriesInfoExtractorServiceImpl implements SeriesInfoExtractorServic
4142
private static final Pattern RELEASE_YEAR_REGEXP =
4243
Pattern.compile("18[4-9][0-9]|19[0-9]{2}|20[0-9]{2}");
4344

45+
// Regular expression matches number of the stamps in a series (from 1 to 99).
46+
private static final Pattern NUMBER_OF_STAMPS_REGEXP =
47+
Pattern.compile("([1-9][0-9]?) марок");
48+
4449
private final Logger log;
4550
private final CategoryService categoryService;
4651
private final CountryService countryService;
@@ -144,4 +149,15 @@ public Integer extractReleaseYear(String fragment) {
144149
return null;
145150
}
146151

152+
// @todo #781 SeriesInfoExtractorServiceImpl.extractQuantity(): add unit tests
153+
// @todo #781 SeriesInfoExtractorServiceImpl.extractQuantity() respect MAX_STAMPS_IN_SERIES
154+
@Override
155+
public Integer extractQuantity(String fragment) {
156+
Matcher matcher = NUMBER_OF_STAMPS_REGEXP.matcher(fragment);
157+
if (matcher.find()) {
158+
return Integer.valueOf(matcher.group(1));
159+
}
160+
return null;
161+
}
162+
147163
}

‎src/main/java/ru/mystamps/web/service/dto/RawParsedDataDto.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,5 @@ public class RawParsedDataDto {
2727
private final String countryName;
2828
private final String imageUrl;
2929
private final String releaseYear;
30+
private final String quantity;
3031
}

‎src/main/java/ru/mystamps/web/util/extractor/SeriesInfo.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ public class SeriesInfo {
3434
private String countryName;
3535
private String imageUrl;
3636
private String issueDate;
37+
private String quantity;
3738

3839
/**
3940
* Check whether any info about a series is available.
@@ -42,7 +43,8 @@ public boolean isEmpty() {
4243
return categoryName == null
4344
&& countryName == null
4445
&& imageUrl == null
45-
&& issueDate == null;
46+
&& issueDate == null
47+
&& quantity == null;
4648
}
4749

4850
}

‎src/main/java/ru/mystamps/web/util/extractor/SiteParser.java

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ public SeriesInfo parse(String htmlPage) {
136136
info.setCountryName(extractCountry(body));
137137
info.setImageUrl(extractImageUrl(body));
138138
info.setIssueDate(extractIssueDate(body));
139+
info.setQuantity(extractQuantity(body));
139140

140141
return info;
141142
}
@@ -209,4 +210,20 @@ protected String extractIssueDate(Element body) {
209210
return date;
210211
}
211212

213+
protected String extractQuantity(Element body) {
214+
String locator = shortDescriptionLocator;
215+
if (locator == null) {
216+
return null;
217+
}
218+
219+
Element elem = body.selectFirst(locator);
220+
if (elem == null) {
221+
return null;
222+
}
223+
224+
String quantity = elem.text();
225+
LOG.debug("Extracted quantity: '{}'", quantity);
226+
return quantity;
227+
}
228+
212229
}

‎src/main/resources/liquibase/version/0.4.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,5 +44,6 @@
4444
<include file="0.4/2017-12-21--solovyov_catalog.xml" relativeToChangelogFile="true" />
4545
<include file="0.4/2017-12-21--zagorski_catalog.xml" relativeToChangelogFile="true" />
4646
<include file="0.4/2017-12-30--series_import_requests_url_length2.xml" relativeToChangelogFile="true" />
47+
<include file="0.4/2018-01-01--add_quantity_to_parsed_data.xml" relativeToChangelogFile="true" />
4748

4849
</databaseChangeLog>

‎src/main/resources/liquibase/version/0.4/2018-01-01--add_quantity_to_parsed_data.xml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<databaseChangeLog
3+
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
6+
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd">
7+
8+
<changeSet id="add-quantity-field-to-series_import_parsed_data" author="php-coder" context="scheme">
9+
10+
<addColumn tableName="series_import_parsed_data">
11+
<column name="quantity" type="INTEGER" afterColumn="image_url" />
12+
</addColumn>
13+
14+
</changeSet>
15+
16+
</databaseChangeLog>

‎src/main/resources/sql/series_import_request_dao_queries.properties

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ INSERT \
8888
, country_id \
8989
, image_url \
9090
, release_year \
91+
, quantity \
9192
, created_at \
9293
, updated_at \
9394
) \
@@ -97,6 +98,7 @@ VALUES \
9798
, :country_id \
9899
, :image_url \
99100
, :release_year \
101+
, :quantity \
100102
, :created_at \
101103
, :updated_at \
102104
)
@@ -110,6 +112,7 @@ series_import_requests.find_parsed_data_by_request_id = \
110112
, count.slug AS country_slug \
111113
, CASE WHEN 'ru' = :lang THEN COALESCE(count.name_ru, count.name) ELSE count.name END AS country_name \
112114
, pd.release_year \
115+
, pd.quantity \
113116
FROM series_import_parsed_data pd \
114117
LEFT JOIN categories cat \
115118
ON cat.id = pd.category_id \

‎src/test/groovy/ru/mystamps/web/service/SeriesImportServiceImplTest.groovy

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -410,7 +410,8 @@ class SeriesImportServiceImplTest extends Specification {
410410
Random.categoryName(),
411411
Random.countryName(),
412412
null, /* imageUrl */
413-
Random.issueYear().toString()
413+
Random.issueYear().toString(),
414+
Random.quantity().toString()
414415
)
415416
and:
416417
extractorService.extractCategory(_ as String) >> Collections.emptyList()
@@ -434,7 +435,8 @@ class SeriesImportServiceImplTest extends Specification {
434435
Random.categoryName(),
435436
Random.countryName(),
436437
expectedImageUrl,
437-
Random.issueYear().toString()
438+
Random.issueYear().toString(),
439+
Random.quantity().toString()
438440
)
439441
and:
440442
extractorService.extractCategory(_ as String) >> Collections.emptyList()
@@ -464,12 +466,14 @@ class SeriesImportServiceImplTest extends Specification {
464466
List<Integer> expectedCountryIds = Random.listOfIntegers()
465467
Integer expectedCategoryId = expectedCategoryIds.get(0)
466468
Integer expectedCountryId = expectedCountryIds.get(0)
469+
Integer expectedQuantity = Random.quantity()
467470
and:
468471
RawParsedDataDto parsedData = new RawParsedDataDto(
469472
expectedCategoryName,
470473
expectedCountryName,
471474
Random.url(),
472-
expectedReleaseYear.toString()
475+
expectedReleaseYear.toString(),
476+
expectedQuantity.toString()
473477
)
474478
when:
475479
service.saveParsedData(expectedRequestId, parsedData)
@@ -479,13 +483,16 @@ class SeriesImportServiceImplTest extends Specification {
479483
1 * extractorService.extractCountry(expectedCountryName) >> expectedCountryIds
480484
and:
481485
1 * extractorService.extractReleaseYear(expectedReleaseYear.toString()) >> expectedReleaseYear
486+
and:
487+
1 * extractorService.extractQuantity(expectedQuantity.toString()) >> expectedQuantity
482488
and:
483489
1 * seriesImportDao.addParsedContent(
484490
expectedRequestId,
485491
{ SaveParsedDataDbDto saveParsedData ->
486492
assert saveParsedData?.categoryId == expectedCategoryId
487493
assert saveParsedData?.countryId == expectedCountryId
488494
assert saveParsedData?.releaseYear == expectedReleaseYear
495+
assert saveParsedData?.quantity == expectedQuantity
489496
return true
490497
}
491498
)

‎src/test/java/ru/mystamps/web/service/TestObjects.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,8 @@ public static ParsedDataDto createParsedDataDto() {
202202
new LinkEntityDto(Random.id(), categorySlug, categoryName),
203203
new LinkEntityDto(Random.id(), countrySlug, countryName),
204204
Random.url(),
205-
Random.issueYear()
205+
Random.issueYear(),
206+
Random.quantity()
206207
);
207208
}
208209

@@ -211,7 +212,8 @@ public static RawParsedDataDto createRawParsedDataDto() {
211212
Random.categoryName(),
212213
Random.countryName(),
213214
Random.url(),
214-
Random.issueYear().toString()
215+
Random.issueYear().toString(),
216+
Random.quantity().toString()
215217
);
216218
}
217219

‎src/test/java/ru/mystamps/web/tests/Random.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,13 @@ public static Integer issueYear() {
144144
return between(ValidationRules.MIN_RELEASE_YEAR, Year.now().getValue()).integer();
145145
}
146146

147+
public static Integer quantity() {
148+
return between(
149+
ValidationRules.MIN_STAMPS_IN_SERIES,
150+
ValidationRules.MAX_STAMPS_IN_SERIES
151+
).integer();
152+
}
153+
147154
public static String importRequestStatus() {
148155
return sample(STATUSES);
149156
}

‎src/test/java/ru/mystamps/web/util/extractor/SiteParserTest.java

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -636,6 +636,44 @@ public void extractIssueDateShouldReturnTextOfShortDescriptionLocator() {
636636
assertThat(msg, date, equalTo(expectedDate));
637637
}
638638

639+
//
640+
// Tests for extractQuantity()
641+
//
642+
643+
@Test
644+
public void extractQuantityShouldReturnNullWhenShortDescriptionLocatorIsNotSet() {
645+
parser.setShortDescriptionLocator(null);
646+
Element doc = createEmptyDocument();
647+
648+
String quantity = parser.extractQuantity(doc);
649+
650+
assertThat(quantity, is(nullValue()));
651+
}
652+
653+
@Test
654+
public void extractQuantityShouldReturnNullWhenElementNotFound() {
655+
parser.setShortDescriptionLocator(Random.jsoupLocator());
656+
Element doc = createEmptyDocument();
657+
658+
String quantity = parser.extractQuantity(doc);
659+
660+
assertThat(quantity, is(nullValue()));
661+
}
662+
663+
@Test
664+
public void extractQuantityShouldReturnTextOfShortDescriptionLocator() {
665+
parser.setShortDescriptionLocator("#desc");
666+
667+
String expectedQuantity = Random.quantity().toString();
668+
String html = String.format("<div id='desc'>%s</div>", expectedQuantity);
669+
Element doc = createDocumentFromText(html);
670+
671+
String quantity = parser.extractQuantity(doc);
672+
673+
String msg = String.format("couldn't extract quantity from '%s'", doc);
674+
assertThat(msg, quantity, equalTo(expectedQuantity));
675+
}
676+
639677
private static String describe(SiteParser parser) {
640678
StringBuilder sb = new StringBuilder();
641679
sb.append("SiteParser[name=")

0 commit comments

Comments
 (0)