Revision 40d1f836
Added by Andreas Müller almost 4 years ago
| app-import/src/main/java/eu/etaxonomy/cdm/app/cyprus/CyprusImagesActivator.java | ||
|---|---|---|
| 66 | 66 |
import eu.etaxonomy.cdm.model.taxon.TaxonBase; |
| 67 | 67 |
|
| 68 | 68 |
/** |
| 69 |
* Creates CDM Media from images stored in the given path. |
|
| 70 |
* |
|
| 71 |
* Note: Currently adapted to also change from Scaler IIF API to default Scaler API. |
|
| 72 |
* Note2: updateMetadata() still needs to be adapted to support 3 MediaRepresentations |
|
| 73 |
* |
|
| 69 | 74 |
* @author a.mueller |
| 70 | 75 |
* @since 05.2017 |
| 71 | 76 |
*/ |
| 72 | 77 |
public class CyprusImagesActivator {
|
| 73 | 78 |
private static final Logger logger = Logger.getLogger(CyprusImagesActivator.class); |
| 74 | 79 |
|
| 75 |
|
|
| 76 |
static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_cyprus(); |
|
| 80 |
static final ICdmDataSource cdmDestination = CdmDestinations.local_cyprus(); |
|
| 81 |
// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_cyprus();
|
|
| 77 | 82 |
// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_production_cyprus(); |
| 78 | 83 |
|
| 79 | 84 |
static boolean testOnly = false; |
| ... | ... | |
| 82 | 87 |
static boolean forceUpdate = false; |
| 83 | 88 |
|
| 84 | 89 |
private static final String path = "//media/digitalimages/EditWP6/Zypern/photos/"; |
| 85 |
private static final String urlPath = "http://media.bgbm.org/erez/erez?src=EditWP6/zypern/photos/"; |
|
| 90 |
private static final String oldUrlPath = "https://pictures.bgbm.org/digilib/Scaler/IIIF/Cyprus!"; |
|
| 91 |
private static final String newUrlPath = "https://pictures.bgbm.org/digilib/Scaler?fn=Cyprus/"; |
|
| 92 |
private static final String oldPostfix = "/full/full/0/default.jpg"; |
|
| 93 |
private static final String newPostfix = "&mo=file"; |
|
| 94 |
private static final String mediumPostfix ="&mo=fit&dw=400&dh=400"; |
|
| 95 |
private static final String smallPostfix ="&mo=fit&dw=200&dh=200"; |
|
| 86 | 96 |
|
| 87 | 97 |
private ImportDeduplicationHelper<SimpleExcelTaxonImportState<?>> deduplicationHelper; |
| 88 | 98 |
|
| ... | ... | |
| 100 | 110 |
String regEx = "([A-Z][a-z]+_[a-z\\-]{3,}(?:_s_[a-z\\-]{3,})?)_[A-F]\\d{1,2}\\.(?:jpg|JPG)";
|
| 101 | 111 |
Pattern pattern = Pattern.compile(regEx); |
| 102 | 112 |
|
| 103 |
for (String fileName : fileList){
|
|
| 113 |
String start = "O"; //O |
|
| 114 |
String end = "Q"; //Q |
|
| 115 |
String startLetter = ""; |
|
| 104 | 116 |
|
| 117 |
for (String fileName : fileList){
|
|
| 118 |
if(fileName.compareToIgnoreCase(start) < 0 || fileName.compareToIgnoreCase(end) >= 0){
|
|
| 119 |
continue; |
|
| 120 |
} |
|
| 105 | 121 |
Matcher matcher = pattern.matcher(fileName); |
| 106 |
if (matcher.matches()){
|
|
| 122 |
if (matcher.matches() ){
|
|
| 107 | 123 |
// System.out.println(fileName); |
| 124 |
if (!fileName.substring(0,3).equals(startLetter)){
|
|
| 125 |
startLetter = fileName.substring(0,3); |
|
| 126 |
System.out.println(startLetter); |
|
| 127 |
} |
|
| 108 | 128 |
String taxonName = matcher.group(1); |
| 109 | 129 |
taxonName = taxonName.replace("_s_", " subsp. ").replace("_", " ");
|
| 110 | 130 |
Taxon taxon = getAcceptedTaxon(app, taxonName); |
| ... | ... | |
| 114 | 134 |
logger.warn("Taxon not found: " + taxonName);
|
| 115 | 135 |
} |
| 116 | 136 |
}else{
|
| 117 |
handleTaxon(app, taxon, fileName); |
|
| 137 |
try {
|
|
| 138 |
handleTaxon(app, taxon, fileName); |
|
| 139 |
} catch (Exception e) {
|
|
| 140 |
logger.error("Unhandled exception ("+e.getMessage()+") when reading file " + fileName +". File not imported: ");
|
|
| 141 |
e.printStackTrace(); |
|
| 142 |
} |
|
| 118 | 143 |
} |
| 119 | 144 |
}else{
|
| 120 | 145 |
if (!fileName.matches("(?:\\.erez|Thumbs\\.db.*|zypern_.*|__Keywords_template\\.txt)")){
|
| 121 | 146 |
logger.warn("Incorrect filename:" + fileName);
|
| 147 |
}else{
|
|
| 148 |
System.out.println("Not clear yet: " + fileName);
|
|
| 122 | 149 |
} |
| 123 | 150 |
} |
| 124 | 151 |
} |
| ... | ... | |
| 132 | 159 |
} |
| 133 | 160 |
|
| 134 | 161 |
private void handleTaxon(CdmApplicationController app, Taxon taxon, String fileName) {
|
| 135 |
Set<String> urlStr = getAllExistingUrls(taxon); |
|
| 136 |
String fullName = urlPath + fileName; |
|
| 137 |
if (urlStr.contains(fullName)){
|
|
| 162 |
Map<String, Media> existingUrls = getAllExistingUrls(taxon); |
|
| 163 |
String pathToOldImage = oldUrlPath + fileName + oldPostfix; |
|
| 164 |
|
|
| 165 |
String pathToFullImage = newUrlPath + fileName + newPostfix; |
|
| 166 |
String pathToMediumImage = newUrlPath + fileName + mediumPostfix; |
|
| 167 |
String pathToSmallImage = newUrlPath + fileName + smallPostfix; |
|
| 168 |
|
|
| 169 |
if (containsAll(existingUrls, pathToFullImage, pathToMediumImage, pathToSmallImage)){
|
|
| 138 | 170 |
return; |
| 139 | 171 |
}else{
|
| 140 |
addMedia(app, taxon, fileName); |
|
| 172 |
Media media; |
|
| 173 |
if (containsAny(existingUrls, pathToOldImage, pathToMediumImage, pathToSmallImage)){
|
|
| 174 |
media = getExistingMedia(existingUrls, pathToOldImage, pathToMediumImage, pathToSmallImage); |
|
| 175 |
if (media == null){
|
|
| 176 |
return; |
|
| 177 |
}else if (media.getAllTitles().isEmpty()){
|
|
| 178 |
media.setTitleCache(null, false); |
|
| 179 |
media.putTitle(Language.LATIN(), fileName); |
|
| 180 |
} |
|
| 181 |
}else{
|
|
| 182 |
media = Media.NewInstance(); |
|
| 183 |
makeMetaData(media, fileName, false); |
|
| 184 |
makeTitle(media, fileName, false); |
|
| 185 |
if (!testOnly){
|
|
| 186 |
makeTextData(fileName, media, taxon); |
|
| 187 |
} |
|
| 188 |
} |
|
| 189 |
fillMediaWithAllRepresentations(media, pathToFullImage, pathToMediumImage, pathToSmallImage, pathToOldImage); |
|
| 141 | 190 |
} |
| 142 | 191 |
} |
| 143 | 192 |
|
| 144 |
/** |
|
| 145 |
* @param app |
|
| 146 |
* @param taxon |
|
| 147 |
* @param fileName |
|
| 148 |
*/ |
|
| 149 |
private void addMedia(CdmApplicationController app, Taxon taxon, String fileName) {
|
|
| 150 |
try {
|
|
| 151 |
String fullName = urlPath + fileName; |
|
| 152 |
Media media = getImageMedia(fullName, null, true); |
|
| 153 |
makeMetaData(media, fileName, false); |
|
| 154 |
makeTitle(media, fileName, false); |
|
| 155 |
if (!testOnly){
|
|
| 156 |
makeTextData(fileName, media, taxon); |
|
| 193 |
private Media getExistingMedia(Map<String, Media> existingUrls, String pathToFullImage, String pathToMediumImage, |
|
| 194 |
String pathToSmallImage) {
|
|
| 195 |
Set<Media> result = new HashSet<>(); |
|
| 196 |
for(String existingUrl : existingUrls.keySet()){
|
|
| 197 |
if (existingUrl.equals(pathToFullImage) || existingUrl.equals(pathToMediumImage) || |
|
| 198 |
existingUrl.equals(pathToSmallImage)){
|
|
| 199 |
result.add(existingUrls.get(existingUrl)); |
|
| 157 | 200 |
} |
| 158 |
|
|
| 159 |
} catch (Exception e) {
|
|
| 160 |
e.printStackTrace(); |
|
| 161 |
return; |
|
| 162 | 201 |
} |
| 202 |
if (result.isEmpty()){
|
|
| 203 |
logger.warn("Media for existing URL not found. This should not happen.");
|
|
| 204 |
return null; |
|
| 205 |
}else if (result.size() > 1){
|
|
| 206 |
logger.warn("Existing URLs have more than 1 Media. This should not happen.");
|
|
| 207 |
return null; |
|
| 208 |
}else{
|
|
| 209 |
return result.iterator().next(); |
|
| 210 |
} |
|
| 211 |
} |
|
| 212 |
|
|
| 213 |
/** |
|
| 214 |
* <code>true</code> if all 3 paths exist in the URL set |
|
| 215 |
*/ |
|
| 216 |
private boolean containsAll(Map<String, Media> existingUrlMap, String pathToFullImage, String pathToMediumImage, |
|
| 217 |
String pathToSmallImage) {
|
|
| 218 |
Set<String> existingUrls = existingUrlMap.keySet(); |
|
| 219 |
return existingUrls.contains(pathToFullImage) && existingUrls.contains(pathToMediumImage) |
|
| 220 |
&& existingUrls.contains(pathToSmallImage); |
|
| 163 | 221 |
} |
| 164 | 222 |
|
| 165 | 223 |
/** |
| 166 |
* @param media |
|
| 167 |
* @param fileName |
|
| 168 |
* @param b |
|
| 224 |
* <code>true</code> if any of the 3 paths exists in the URL set |
|
| 169 | 225 |
*/ |
| 226 |
private boolean containsAny(Map<String, Media> existingUrlMap, String pathToFullImage, String pathToMediumImage, |
|
| 227 |
String pathToSmallImage) {
|
|
| 228 |
Set<String> existingUrls = existingUrlMap.keySet(); |
|
| 229 |
return existingUrls.contains(pathToFullImage) || existingUrls.contains(pathToMediumImage) |
|
| 230 |
|| existingUrls.contains(pathToSmallImage); |
|
| 231 |
} |
|
| 232 |
|
|
| 170 | 233 |
private void makeTitle(Media media, String fileName, boolean updateOnly) {
|
| 171 | 234 |
String title = fileName.replace("_s_"," subsp. ")
|
| 172 | 235 |
.replace("_"," ").replace(".jpg","").replace(".JPG","");
|
| ... | ... | |
| 329 | 392 |
textData.addMedia(media); |
| 330 | 393 |
} |
| 331 | 394 |
|
| 395 |
private void fillMediaWithAllRepresentations(Media media, String fullPath, String mediumPath, String smallPath, String oldFullPath){
|
|
| 396 |
Set<MediaRepresentation> existingRepresentations = new HashSet<>(media.getRepresentations()); |
|
| 397 |
makeMediaRepresentation(oldFullPath, media, existingRepresentations, fullPath); |
|
| 398 |
makeMediaRepresentation(mediumPath, media, existingRepresentations, null); |
|
| 399 |
makeMediaRepresentation(smallPath, media, existingRepresentations, null); |
|
| 400 |
if(!existingRepresentations.isEmpty()){
|
|
| 401 |
logger.warn("Media contains existing representations which are not contained in the 3 paths: " + media.getTitleCache());
|
|
| 402 |
} |
|
| 403 |
} |
|
| 404 |
|
|
| 405 |
private void makeMediaRepresentation(String uriString, Media media, |
|
| 406 |
Set<MediaRepresentation> existingRepresentations, String replaceUri) {
|
|
| 407 |
MediaRepresentation existingMediaRep = getExistingMediaRepresentation(uriString, existingRepresentations); |
|
| 408 |
boolean readMediaData = true; |
|
| 409 |
MediaRepresentation newMediaRep = makeMediaRepresentation(replaceUri != null? replaceUri : uriString, readMediaData); |
|
| 410 |
if (existingMediaRep == null){
|
|
| 411 |
media.addRepresentation(newMediaRep); |
|
| 412 |
}else{
|
|
| 413 |
existingRepresentations.remove(existingMediaRep); |
|
| 414 |
mergeToExistingRepresentation(existingMediaRep, newMediaRep); |
|
| 415 |
} |
|
| 416 |
} |
|
| 417 |
|
|
| 418 |
private void mergeToExistingRepresentation(MediaRepresentation existingMediaRep, MediaRepresentation newMediaRep) {
|
|
| 419 |
existingMediaRep.setMimeType(newMediaRep.getMimeType()); |
|
| 420 |
existingMediaRep.setSuffix(newMediaRep.getSuffix()); |
|
| 421 |
if(!existingMediaRep.getParts().isEmpty() && !newMediaRep.getParts().isEmpty()){
|
|
| 422 |
MediaRepresentationPart existingPart = existingMediaRep.getParts().iterator().next(); |
|
| 423 |
ImageFile newPart = (ImageFile)newMediaRep.getParts().iterator().next(); |
|
| 424 |
if(existingPart.isInstanceOf(ImageFile.class)){
|
|
| 425 |
ImageFile existingImage = CdmBase.deproxy(existingPart, ImageFile.class); |
|
| 426 |
existingImage.setHeight(newPart.getHeight()); |
|
| 427 |
existingImage.setWidth(newPart.getWidth()); |
|
| 428 |
}else{
|
|
| 429 |
logger.warn("MediaRepresentationPart was not of type ImageFile. Height and width not merged: " + existingPart.getUri());
|
|
| 430 |
} |
|
| 431 |
existingPart.setSize(newPart.getSize()); |
|
| 432 |
existingPart.setUri(newPart.getUri()); |
|
| 433 |
} |
|
| 434 |
} |
|
| 435 |
|
|
| 436 |
private MediaRepresentation getExistingMediaRepresentation(String uriString, |
|
| 437 |
Set<MediaRepresentation> existingRepresentations) {
|
|
| 438 |
for (MediaRepresentation rep : existingRepresentations){
|
|
| 439 |
for (MediaRepresentationPart part : rep.getParts()){
|
|
| 440 |
if (part.getUri() != null && part.getUri().toString().equals(uriString)){
|
|
| 441 |
return rep; |
|
| 442 |
} |
|
| 443 |
} |
|
| 444 |
} |
|
| 445 |
return null; |
|
| 446 |
} |
|
| 447 |
|
|
| 332 | 448 |
/** |
| 333 | 449 |
* Creates |
| 334 | 450 |
* @see #READ_MEDIA_DATA |
| ... | ... | |
| 341 | 457 |
} else {
|
| 342 | 458 |
uriString = uriString.replace(" ", "%20"); //replace whitespace
|
| 343 | 459 |
try {
|
| 344 |
ImageInfo imageInfo = null; |
|
| 345 |
URI uri = new URI(uriString); |
|
| 346 |
|
|
| 347 |
try {
|
|
| 348 |
if (readMediaData){
|
|
| 349 |
logger.info("Read media data from: " + uri);
|
|
| 350 |
imageInfo = ImageInfo.NewInstance(uri, 0); |
|
| 351 |
} |
|
| 352 |
} catch (Exception e) {
|
|
| 353 |
String message = "An error occurred when trying to read image meta data for " + uri.toString() + ": " + e.getMessage(); |
|
| 354 |
logger.warn(message); |
|
| 355 |
} |
|
| 356 |
ImageFile imageFile = ImageFile.NewInstance(uri, null, imageInfo); |
|
| 357 |
|
|
| 358 |
MediaRepresentation representation = MediaRepresentation.NewInstance(); |
|
| 359 |
|
|
| 360 |
if(imageInfo != null){
|
|
| 361 |
representation.setMimeType(imageInfo.getMimeType()); |
|
| 362 |
representation.setSuffix(imageInfo.getSuffix()); |
|
| 363 |
} |
|
| 364 |
representation.addRepresentationPart(imageFile); |
|
| 460 |
MediaRepresentation representation = makeMediaRepresentation(uriString, readMediaData); |
|
| 365 | 461 |
Media media = Media.NewInstance(); |
| 366 | 462 |
media.addRepresentation(representation); |
| 367 | 463 |
|
| ... | ... | |
| 398 | 494 |
} |
| 399 | 495 |
} |
| 400 | 496 |
|
| 401 |
/** |
|
| 402 |
* @param taxon |
|
| 403 |
* @return |
|
| 404 |
*/ |
|
| 405 |
private Set<String> getAllExistingUrls(Taxon taxon) {
|
|
| 406 |
Set<String> result = new HashSet<>(); |
|
| 497 |
private MediaRepresentation makeMediaRepresentation(String uriString, boolean readMediaData) {
|
|
| 498 |
|
|
| 499 |
uriString = uriString.replace(" ", "%20"); //replace whitespace
|
|
| 500 |
ImageInfo imageInfo = null; |
|
| 501 |
URI uri; |
|
| 502 |
try {
|
|
| 503 |
uri = new URI(uriString); |
|
| 504 |
} catch (URISyntaxException e1) {
|
|
| 505 |
logger.error("Malformed URI. Could not create media representation: " + uriString);
|
|
| 506 |
return null; |
|
| 507 |
} |
|
| 508 |
try {
|
|
| 509 |
if (readMediaData){
|
|
| 510 |
logger.info("Read media data from: " + uri);
|
|
| 511 |
imageInfo = ImageInfo.NewInstance(uri, 0); |
|
| 512 |
} |
|
| 513 |
} catch (Exception e) {
|
|
| 514 |
try {
|
|
| 515 |
//try again |
|
| 516 |
imageInfo = ImageInfo.NewInstance(uri, 0); |
|
| 517 |
} catch (Exception e1) {
|
|
| 518 |
String message = "An error occurred when trying to read image meta data for " + uri.toString() + ": " + e1.getMessage(); |
|
| 519 |
e1.printStackTrace(); |
|
| 520 |
logger.warn(message); |
|
| 521 |
} |
|
| 522 |
} |
|
| 523 |
ImageFile imageFile = ImageFile.NewInstance(uri, null, imageInfo); |
|
| 524 |
|
|
| 525 |
MediaRepresentation representation = MediaRepresentation.NewInstance(); |
|
| 526 |
|
|
| 527 |
if(imageInfo != null){
|
|
| 528 |
representation.setMimeType(imageInfo.getMimeType()); |
|
| 529 |
representation.setSuffix(imageInfo.getSuffix()); |
|
| 530 |
} |
|
| 531 |
representation.addRepresentationPart(imageFile); |
|
| 532 |
return representation; |
|
| 533 |
} |
|
| 534 |
|
|
| 535 |
private Map<String, Media> getAllExistingUrls(Taxon taxon) {
|
|
| 536 |
Map<String, Media> result = new HashMap<>(); |
|
| 407 | 537 |
Set<TaxonDescription> descriptions = taxon.getDescriptions(); |
| 408 | 538 |
for (TaxonDescription td : descriptions){
|
| 409 | 539 |
if (td.isImageGallery()){
|
| ... | ... | |
| 416 | 546 |
URI uri = part.getUri(); |
| 417 | 547 |
if (uri != null){
|
| 418 | 548 |
String uriStr = uri.toString(); |
| 419 |
result.add(uriStr);
|
|
| 549 |
result.put(uriStr, media);
|
|
| 420 | 550 |
} |
| 421 | 551 |
} |
| 422 | 552 |
} |
| 423 | 553 |
} |
| 424 |
|
|
| 425 | 554 |
} |
| 426 | 555 |
} |
| 427 | 556 |
} |
| ... | ... | |
| 513 | 642 |
List<Media> list = app.getMediaService().list(Media.class, null, null, null, null); |
| 514 | 643 |
for (Media media : list){
|
| 515 | 644 |
String fileName = getUrlStringForMedia(media); |
| 516 |
if (fileName.startsWith(urlPath)){
|
|
| 517 |
fileName = fileName.replace(urlPath, ""); |
|
| 645 |
if (fileName.startsWith(newUrlPath)){
|
|
| 646 |
//TODO not yet adapted to new image server URLs |
|
| 647 |
fileName = fileName.replace(newUrlPath, ""); |
|
| 518 | 648 |
if (fileName.equals("Acinos_exiguus_C1.jpg")){ //for debugging only
|
| 519 | 649 |
// System.out.println(fileName); |
| 520 | 650 |
makeMetaData(media, fileName, true); |
| ... | ... | |
| 538 | 668 |
URI uri = part.getUri(); |
| 539 | 669 |
if (uri != null){
|
| 540 | 670 |
if (result != null){
|
| 671 |
//TODO this still needs to be adapted to the 3 representations of media |
|
| 541 | 672 |
logger.warn("More than 1 uri exists for media "+ media.getId());
|
| 542 | 673 |
}else{
|
| 543 | 674 |
result = uri.toString(); |
Also available in: Unified diff
ref #9132 Cyprus images updated for IIIF and Scaler API