app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelNameFactsImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9 package eu.etaxonomy.cdm.io.berlinModel.in;
  10
  11 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.NAME_FACT_ALSO_PUBLISHED_IN;
  12 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.NAME_FACT_BIBLIOGRAPHY;
  13 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.NAME_FACT_PROTOLOGUE;
  14
  15 import java.io.File;
  16 import java.io.IOException;
  17 import java.net.MalformedURLException;
  18 import java.net.URISyntaxException;
  19 import java.net.URL;
  20 import java.sql.ResultSet;
  21 import java.sql.SQLException;
  22 import java.util.HashMap;
  23 import java.util.HashSet;
  24 import java.util.Map;
  25 import java.util.Set;
  26
  27 import org.apache.commons.lang3.StringUtils;
  28 import org.apache.http.HttpException;
  29 import org.apache.logging.log4j.LogManager;
  30 import org.apache.logging.log4j.Logger;
  31 import org.springframework.stereotype.Component;
  32
  33 import eu.etaxonomy.cdm.api.service.media.MediaInfoFileReader;
  34 import eu.etaxonomy.cdm.common.CdmUtils;
  35 import eu.etaxonomy.cdm.common.URI;
  36 import eu.etaxonomy.cdm.common.media.CdmImageInfo;
  37 import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelNameFactsImportValidator;
  38 import eu.etaxonomy.cdm.io.common.IOValidator;
  39 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  40 import eu.etaxonomy.cdm.model.common.CdmBase;
  41 import eu.etaxonomy.cdm.model.common.Language;
  42 import eu.etaxonomy.cdm.model.description.Feature;
  43 import eu.etaxonomy.cdm.model.description.TaxonNameDescription;
  44 import eu.etaxonomy.cdm.model.description.TextData;
  45 import eu.etaxonomy.cdm.model.media.ExternalLinkType;
  46 import eu.etaxonomy.cdm.model.media.ImageFile;
  47 import eu.etaxonomy.cdm.model.media.Media;
  48 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
  49 import eu.etaxonomy.cdm.model.media.MediaRepresentationPart;
  50 import eu.etaxonomy.cdm.model.name.TaxonName;
  51 import eu.etaxonomy.cdm.model.reference.OriginalSourceType;
  52 import eu.etaxonomy.cdm.model.reference.Reference;
  53
  54 /**
  55  * @author a.mueller
  56  * @since 20.03.2008
  57  */
  58 @Component
  59 public class BerlinModelNameFactsImport  extends BerlinModelImportBase  {
  60     private static final long serialVersionUID = 4174085686074314138L;
  61
  62     private static final Logger logger = LogManager.getLogger();
  63
  64         public static final String NAMESPACE = "NameFact";
  65
  66         /**
  67          * write info message after modCount iterations
  68          */
  69         private int modCount = 500;
  70         private static final String pluralString = "name facts";
  71         private static final String dbTableName = "NameFact";
  72
  73
  74         public BerlinModelNameFactsImport(){
  75                 super(dbTableName, pluralString);
  76         }
  77
  78
  79         @Override
  80         protected String getIdQuery(BerlinModelImportState state) {
  81                 if (isNotBlank(state.getConfig().getNameIdTable())){
  82                         String result = super.getIdQuery(state);
  83                         result += " WHERE ptNameFk IN (SELECT NameId FROM " + state.getConfig().getNameIdTable() + ")";
  84                         if (state.getConfig().isEuroMed()){
  85                             result += " AND NOT (NameFactCategoryFk = 3 AND NameFactRefFk = 8500000) ";  //#7796#note-11
  86                         }
  87                         return result;
  88                 }else{
  89                         return super.getIdQuery(state);
  90                 }
  91         }
  92
  93         @Override
  94         protected String getRecordQuery(BerlinModelImportConfigurator config) {
  95                 String strQuery =
  96                         " SELECT NameFact.*, Name.NameID as nameId, NameFactCategory.NameFactCategory " +
  97                         " FROM NameFact INNER JOIN " +
  98                 " Name ON NameFact.PTNameFk = Name.NameId  INNER JOIN "+
  99                 " NameFactCategory ON NameFactCategory.NameFactCategoryID = NameFact.NameFactCategoryFK " +
 100             " WHERE (NameFactId IN ("+ ID_LIST_TOKEN+") )";
 101                 return strQuery;
 102         }
 103
 104         @Override
 105     public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, BerlinModelImportState state) {
 106                 boolean success = true ;
 107                 BerlinModelImportConfigurator config = state.getConfig();
 108                 Set<TaxonName> nameToSave = new HashSet<>();
 109                 @SuppressWarnings("unchecked")
 110         Map<String, TaxonName> nameMap = partitioner.getObjectMap(BerlinModelTaxonNameImport.NAMESPACE);
 111                 @SuppressWarnings("unchecked")
 112         Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);
 113
 114                 ResultSet rs = partitioner.getResultSet();
 115
 116                 Reference sourceRef = state.getTransactionalSourceReference();
 117                 try {
 118                         int i = 0;
 119                         //for each reference
 120                         while (rs.next() && (config.getMaximumNumberOfNameFacts() == 0 || i < config.getMaximumNumberOfNameFacts())){
 121
 122                                 if ((i++ % modCount) == 0  && i!= 1 ){ logger.info("NameFacts handled: " + (i-1));}
 123
 124                                 int nameFactId = rs.getInt("nameFactId");
 125                                 int nameId = rs.getInt("nameId");
 126                                 Object nameFactRefFkObj = rs.getObject("nameFactRefFk");
 127                                 String nameFactRefDetail = rs.getString("nameFactRefDetail");
 128
 129                                 String category = CdmUtils.Nz(rs.getString("NameFactCategory"));
 130                                 String nameFact = CdmUtils.Nz(rs.getString("nameFact"));
 131
 132                                 TaxonName taxonName = nameMap.get(String.valueOf(nameId));
 133                                 String nameFactRefFk = String.valueOf(nameFactRefFkObj);
 134                                 Reference citation = refMap.get(nameFactRefFk);
 135
 136                                 if (taxonName != null){
 137                                         //PROTOLOGUE
 138                                         if (category.equalsIgnoreCase(NAME_FACT_PROTOLOGUE)){
 139
 140                                             String uriString = config.getMediaUrl() + "/" + nameFact;
 141                                                 try{
 142                                                     //this depends on specific project implementation, maybe also config.getMediaPath() is important
 143                                                     URI uri = URI.create(uriString);
 144                                                         taxonName.addProtologue(uri, null, ExternalLinkType.Unknown);
 145                                                 }catch(IllegalArgumentException e){
 146                                                         logger.warn("Incorrect protologue URI: " + uriString);
 147                                                         success = false;
 148                                                 }
 149                                                 //end NAME_FACT_PROTOLOGUE
 150                                         }else if (category.equalsIgnoreCase(NAME_FACT_ALSO_PUBLISHED_IN)){
 151                                                 if (StringUtils.isNotBlank(nameFact)){
 152                                                         TaxonNameDescription description = TaxonNameDescription.NewInstance();
 153                                                         TextData additionalPublication = TextData.NewInstance(Feature.ADDITIONAL_PUBLICATION());
 154                                                         //TODO language
 155                                                         Language language = Language.DEFAULT();
 156                                                         additionalPublication.putText(language, nameFact);
 157                                                         additionalPublication.addSource(OriginalSourceType.Import, String.valueOf(nameFactId), NAMESPACE, null,null, null, null);
 158                                                         if (citation != null || isNotBlank(nameFactRefDetail)){
 159                                                                 additionalPublication.addSource(OriginalSourceType.PrimaryTaxonomicSource, null, null, citation, nameFactRefDetail, null, null);
 160                                                         }
 161                                                         description.addElement(additionalPublication);
 162                                                         taxonName.addDescription(description);
 163                                                 }
 164                                         }else if (category.equalsIgnoreCase(NAME_FACT_BIBLIOGRAPHY)){
 165                                                 if (isNotBlank(nameFact)){
 166                                                         TaxonNameDescription description = TaxonNameDescription.NewInstance();
 167                                                         TextData bibliography = TextData.NewInstance(Feature.CITATION());
 168                                                         //TODO language
 169                                                         Language language = Language.DEFAULT();
 170                                                         bibliography.putText(language, nameFact);
 171                                                         bibliography.addSource(OriginalSourceType.Import, String.valueOf(nameFactId), NAMESPACE, null,null, null, null);
 172                                                         if (citation != null || isNotBlank(nameFactRefDetail)){
 173                                                                 bibliography.addSource(OriginalSourceType.PrimaryTaxonomicSource, null, null, citation, nameFactRefDetail, null, null);
 174                                                         }
 175                                                         description.addElement(bibliography);
 176                                                         taxonName.addDescription(description);
 177                                                 }
 178                                         }else {
 179                                                 //TODO
 180                                                 logger.warn("NameFactCategory '" + category + "' not yet implemented");
 181                                                 success = false;
 182                                         }
 183
 184                                         //TODO
 185 //                                      DoubtfulFlag    bit        Checked
 186 //                                      PublishFlag      bit        Checked
 187 //                                      Created_When  datetime           Checked
 188 //                                      Updated_When datetime           Checked
 189 //                                      Created_Who    nvarchar(255)    Checked
 190 //                                      Updated_Who  nvarchar(255)    Checked
 191 //                                      Notes      nvarchar(1000)           Checked
 192
 193                                         nameToSave.add(taxonName);
 194                                 }else{
 195                                         //TODO
 196                                         logger.warn("TaxonName for NameFact " + nameFactId + " does not exist in store");
 197                                         success = false;
 198                                 }
 199                                 //put
 200                         }
 201                         if (config.getMaximumNumberOfNameFacts() != 0 && i >= config.getMaximumNumberOfNameFacts() - 1){
 202                                 logger.warn("ONLY " + config.getMaximumNumberOfNameFacts() + " NAMEFACTS imported !!!" )
 203                         ;}
 204                         logger.info("Names to save: " + nameToSave.size());
 205                         getNameService().save(nameToSave);
 206                         return success;
 207                 } catch (SQLException e) {
 208                         logger.error("SQLException:" +  e);
 209                         return false;
 210                 }
 211         }
 212
 213         @Override
 214         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, BerlinModelImportState state) {
 215
 216             String nameSpace;
 217                 Set<String> idSet;
 218                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
 219
 220                 try{
 221                         Set<String> nameIdSet = new HashSet<>();
 222                         Set<String> referenceIdSet = new HashSet<>();
 223                         while (rs.next()){
 224                                 handleForeignKey(rs, nameIdSet, "PTnameFk");
 225                                 handleForeignKey(rs, referenceIdSet, "nameFactRefFk");
 226                         }
 227
 228                         //name map
 229                         nameSpace = BerlinModelTaxonNameImport.NAMESPACE;
 230                         idSet = nameIdSet;
 231                         Map<String, TaxonName> objectMap = getCommonService().getSourcedObjectsByIdInSourceC(TaxonName.class, idSet, nameSpace);
 232                         result.put(nameSpace, objectMap);
 233
 234                         //reference map
 235                         nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
 236                         idSet = referenceIdSet;
 237                         Map<String, Reference> referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace);
 238                         result.put(nameSpace, referenceMap);
 239
 240
 241                 } catch (SQLException e) {
 242                         throw new RuntimeException(e);
 243                 }
 244                 return result;
 245         }
 246
 247         //FIXME gibt es da keine allgemeine Methode in common?
 248         public Media getMedia(String nameFact, URL mediaUrl, File mediaPath){
 249                 if (mediaUrl == null){
 250                         logger.warn("Media Url should not be null");
 251                         return null;
 252                 }
 253                 String mimeTypeTif = "image/tiff";
 254                 String mimeTypeJpg = "image/jpeg";
 255                 String mimeTypePng = "image/png";
 256                 String mimeTypePdf = "application/pdf";
 257                 String suffixTif = "tif";
 258                 String suffixJpg = "jpg";
 259                 String suffixPng = "png";
 260                 String suffixPdf = "pdf";
 261
 262                 String sep = File.separator;
 263                 Integer size = null;
 264
 265                 logger.debug("Getting media for NameFact: " + nameFact);
 266
 267                 Media media = Media.NewInstance();
 268
 269                 String mediaUrlString = mediaUrl.toString();
 270
 271                 //tiff
 272                 String urlStringTif = mediaUrlString + "tif/" + nameFact + "." + suffixTif;
 273                 File file = new File(mediaPath, "tif" + sep + nameFact + "." + suffixTif);
 274                 MediaRepresentation representationTif = MediaRepresentation.NewInstance(mimeTypeTif, suffixTif);
 275                 if (file.exists()){
 276                         representationTif.addRepresentationPart(makeImage(urlStringTif, size, file));
 277                 }
 278                 if(representationTif.getParts().size() > 0){
 279                         media.addRepresentation(representationTif);
 280                 }
 281                 // end tif
 282                 // jpg
 283                 boolean fileExists = true;
 284                 int jpgCount = 0;
 285                 MediaRepresentation representationJpg = MediaRepresentation.NewInstance(mimeTypeJpg, suffixJpg);
 286                 while(fileExists){
 287                         String urlStringJpeg = mediaUrlString + "cmd_jpg/" + nameFact + "_page_000" + jpgCount + "." + suffixJpg;
 288                         file = new File(mediaPath, "cmd_jpg" + sep + nameFact + "_page_000" + jpgCount + "." + suffixJpg);
 289                         jpgCount++;
 290                         if (file.exists()){
 291                                 representationJpg.addRepresentationPart(makeImage(urlStringJpeg, size, file));
 292                         }else{
 293                                 fileExists = false;
 294                         }
 295                 }
 296                 if(representationJpg.getParts().size() > 0){
 297                         media.addRepresentation(representationJpg);
 298                 }
 299                 // end jpg
 300                 //png
 301                 String urlStringPng = mediaUrlString + "png/" + nameFact + "." + suffixPng;
 302                 file = new File(mediaPath, "png" + sep + nameFact + "." + suffixPng);
 303                 MediaRepresentation representationPng = MediaRepresentation.NewInstance(mimeTypePng, suffixPng);
 304                 if (file.exists()){
 305                         representationPng.addRepresentationPart(makeImage(urlStringPng, size, file));
 306                 }else{
 307                         fileExists = true;
 308                         int pngCount = 0;
 309                         while (fileExists){
 310                                 pngCount++;
 311                                 urlStringPng = mediaUrlString + "png/" + nameFact + "00" + pngCount + "." + suffixPng;
 312                                 file = new File(mediaPath, "png" + sep + nameFact + "00" + pngCount + "." + suffixPng);
 313
 314                                 if (file.exists()){
 315                                         representationPng.addRepresentationPart(makeImage(urlStringPng, size, file));
 316                                 }else{
 317                                         fileExists = false;
 318                                 }
 319                         }
 320                 }
 321                 if(representationPng.getParts().size() > 0){
 322                         media.addRepresentation(representationPng);
 323                 }
 324                 //end png
 325         //pdf
 326         String urlStringPdf = mediaUrlString + "pdf/" + nameFact + "." + suffixPdf;
 327         URI uriPdf;
 328                 try {
 329                         uriPdf = new URI(urlStringPdf);
 330                         file = new File(mediaPath, "pdf" + sep + nameFact + "." + suffixPdf);
 331                 MediaRepresentation representationPdf = MediaRepresentation.NewInstance(mimeTypePdf, suffixPdf);
 332                 if (file.exists()){
 333                         representationPdf.addRepresentationPart(MediaRepresentationPart.NewInstance(uriPdf, size));
 334                 }else{
 335                         fileExists = true;
 336                         int pdfCount = 0;
 337                         while (fileExists){
 338                                 pdfCount++;
 339                                 urlStringPdf = mediaUrlString + "pdf/" + nameFact + "00" + pdfCount + "." + suffixPdf;
 340                                 file = new File(mediaPath, "pdf/" + sep + nameFact + "00" + pdfCount + "." + suffixPdf);
 341
 342                                 if (file.exists()){
 343                                         representationPdf.addRepresentationPart(MediaRepresentationPart.NewInstance(uriPdf, size));
 344                                 }else{
 345                                         fileExists = false;
 346                                 }
 347                         }
 348                 }
 349                         if(representationPdf.getParts().size() > 0){
 350                         media.addRepresentation(representationPdf);
 351                 }
 352                 } catch (URISyntaxException e) {
 353                         e.printStackTrace();
 354                         logger.error("URISyntaxException" + urlStringPdf);
 355                 }
 356         //end pdf
 357
 358                 if(logger.isDebugEnabled()){
 359                         for (MediaRepresentation rep : media.getRepresentations()){
 360                                 for (MediaRepresentationPart part : rep.getParts()){
 361                                         logger.debug("in representation: " + part.getUri());
 362                                 }
 363                         }
 364                 }
 365
 366                 return media;
 367         }
 368
 369         private ImageFile makeImage(String imageUri, Integer size, File file){
 370                 CdmImageInfo imageMetaData = null;
 371                 URI uri;
 372                 try {
 373                         uri = new URI(imageUri);
 374                         try {
 375                             //imageMetaData = CdmImageInfo.NewInstance(uri, 0);
 376                                 imageMetaData = MediaInfoFileReader.legacyFactoryMethod(uri)
 377                             .readBaseInfo()
 378                             .getCdmImageInfo();
 379                         } catch (IOException e) {
 380                                 logger.error("IOError reading image metadata." , e);
 381                         } catch (HttpException e) {
 382                                 logger.error("HttpException reading image metadata." , e);
 383                         }
 384                         ImageFile image = ImageFile.NewInstance(uri, size, imageMetaData);
 385                         return image;
 386                 } catch (URISyntaxException e1) {
 387                         logger.warn("URISyntaxException: " + imageUri);
 388                         return null;
 389                 }
 390
 391         }
 392
 393         @Override
 394         protected boolean doCheck(BerlinModelImportState state){
 395                 IOValidator<BerlinModelImportState> validator = new BerlinModelNameFactsImportValidator();
 396                 return validator.validate(state);
 397         }
 398
 399         @Override
 400     protected boolean isIgnore(BerlinModelImportState state){
 401                 return ! state.getConfig().isDoNameFacts();
 402         }
 403
 404         //for testing only
 405         public static void main(String[] args) {
 406
 407                 BerlinModelNameFactsImport nf = new BerlinModelNameFactsImport();
 408
 409                 URL url;
 410                 try {
 411                         url = new URL("http://wp5.e-taxonomy.eu/dataportal/cichorieae/media/protolog/");
 412                         File path = new File("/Volumes/protolog/protolog/");
 413                         if(path.exists()){
 414                                 String fact = "Acanthocephalus_amplexifolius";
 415                                 // make getMedia public for this to work
 416                                 Media media = nf.getMedia(fact, url, path);
 417                                 logger.info(media);
 418                                 for (MediaRepresentation rep : media.getRepresentations()){
 419                                         logger.info(rep.getMimeType());
 420                                         for (MediaRepresentationPart part : rep.getParts()){
 421                                                 logger.info(part.getUri());
 422                                         }
 423                                 }
 424                         }
 425                 } catch (MalformedURLException e) {
 426                         e.printStackTrace();
 427                 }
 428         }
 429 }