app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelNameFactsImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.berlinModel.in;
  11
  12 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.NAME_FACT_ALSO_PUBLISHED_IN;
  13 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.NAME_FACT_BIBLIOGRAPHY;
  14 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.NAME_FACT_PROTOLOGUE;
  15
  16 import java.io.File;
  17 import java.io.IOException;
  18 import java.net.MalformedURLException;
  19 import java.net.URI;
  20 import java.net.URISyntaxException;
  21 import java.net.URL;
  22 import java.sql.ResultSet;
  23 import java.sql.SQLException;
  24 import java.util.HashMap;
  25 import java.util.HashSet;
  26 import java.util.Map;
  27 import java.util.Set;
  28
  29 import org.apache.commons.lang.StringUtils;
  30 import org.apache.http.HttpException;
  31 import org.apache.log4j.Logger;
  32 import org.springframework.stereotype.Component;
  33
  34 import eu.etaxonomy.cdm.common.CdmUtils;
  35 import eu.etaxonomy.cdm.common.media.ImageInfo;
  36 import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelNameFactsImportValidator;
  37 import eu.etaxonomy.cdm.io.common.IOValidator;
  38 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  39 import eu.etaxonomy.cdm.model.agent.Person;
  40 import eu.etaxonomy.cdm.model.common.CdmBase;
  41 import eu.etaxonomy.cdm.model.common.Language;
  42 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  43 import eu.etaxonomy.cdm.model.description.Feature;
  44 import eu.etaxonomy.cdm.model.description.TaxonNameDescription;
  45 import eu.etaxonomy.cdm.model.description.TextData;
  46 import eu.etaxonomy.cdm.model.media.ImageFile;
  47 import eu.etaxonomy.cdm.model.media.Media;
  48 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
  49 import eu.etaxonomy.cdm.model.media.MediaRepresentationPart;
  50 import eu.etaxonomy.cdm.model.name.TaxonName;
  51 import eu.etaxonomy.cdm.model.reference.Reference;
  52
  53
  54 /**
  55  * @author a.mueller
  56  * @created 20.03.2008
  57  * @version 1.0
  58  */
  59 @Component
  60 public class BerlinModelNameFactsImport  extends BerlinModelImportBase  {
  61         private static final Logger logger = Logger.getLogger(BerlinModelNameFactsImport.class);
  62
  63         public static final String NAMESPACE = "NameFact";
  64
  65         /**
  66          * write info message after modCount iterations
  67          */
  68         private int modCount = 500;
  69         private static final String pluralString = "name facts";
  70         private static final String dbTableName = "NameFact";
  71
  72
  73         public BerlinModelNameFactsImport(){
  74                 super(dbTableName, pluralString);
  75         }
  76
  77
  78
  79
  80         @Override
  81         protected String getIdQuery(BerlinModelImportState state) {
  82                 if (StringUtils.isNotEmpty(state.getConfig().getNameIdTable())){
  83                         String result = super.getIdQuery(state);
  84                         result += " WHERE ptNameFk IN (SELECT NameId FROM " + state.getConfig().getNameIdTable() + ")";
  85                         return result;
  86                 }else{
  87                         return super.getIdQuery(state);
  88                 }
  89         }
  90
  91
  92
  93
  94         /* (non-Javadoc)
  95          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
  96          */
  97         @Override
  98         protected String getRecordQuery(BerlinModelImportConfigurator config) {
  99                 String strQuery =
 100                         " SELECT NameFact.*, Name.NameID as nameId, NameFactCategory.NameFactCategory " +
 101                         " FROM NameFact INNER JOIN " +
 102                 " Name ON NameFact.PTNameFk = Name.NameId  INNER JOIN "+
 103                 " NameFactCategory ON NameFactCategory.NameFactCategoryID = NameFact.NameFactCategoryFK " +
 104             " WHERE (NameFactId IN ("+ ID_LIST_TOKEN+") )";
 105                 return strQuery;
 106         }
 107
 108         @Override
 109     public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
 110                 boolean success = true ;
 111                 BerlinModelImportConfigurator config = state.getConfig();
 112                 Set<TaxonName> nameToSave = new HashSet<>();
 113                 Map<String, TaxonName> nameMap = partitioner.getObjectMap(BerlinModelTaxonNameImport.NAMESPACE);
 114                 Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);
 115
 116                 ResultSet rs = partitioner.getResultSet();
 117
 118                 Reference sourceRef = state.getTransactionalSourceReference();
 119                 try {
 120                         int i = 0;
 121                         //for each reference
 122                         while (rs.next() && (config.getMaximumNumberOfNameFacts() == 0 || i < config.getMaximumNumberOfNameFacts())){
 123
 124                                 if ((i++ % modCount) == 0  && i!= 1 ){ logger.info("NameFacts handled: " + (i-1));}
 125
 126                                 int nameFactId = rs.getInt("nameFactId");
 127                                 int nameId = rs.getInt("nameId");
 128                                 Object nameFactRefFkObj = rs.getObject("nameFactRefFk");
 129                                 String nameFactRefDetail = rs.getString("nameFactRefDetail");
 130
 131                                 String category = CdmUtils.Nz(rs.getString("NameFactCategory"));
 132                                 String nameFact = CdmUtils.Nz(rs.getString("nameFact"));
 133
 134                                 TaxonName taxonNameBase = nameMap.get(String.valueOf(nameId));
 135                                 String nameFactRefFk = String.valueOf(nameFactRefFkObj);
 136                                 Reference citation = refMap.get(nameFactRefFk);
 137
 138                                 if (taxonNameBase != null){
 139                                         //PROTOLOGUE
 140                                         if (category.equalsIgnoreCase(NAME_FACT_PROTOLOGUE)){
 141                                                 //Reference ref = (Reference)taxonNameBase.getNomenclaturalReference();
 142                                                 //ref = Book.NewInstance();
 143                                                 try{
 144                                                         Media media = getMedia(nameFact, config.getMediaUrl(), config.getMediaPath());
 145                                                         if (media.getRepresentations().size() > 0){
 146                                                                 TaxonNameDescription description = TaxonNameDescription.NewInstance();
 147                                                                 TextData protolog = TextData.NewInstance(Feature.PROTOLOGUE());
 148                                                                 protolog.addMedia(media);
 149                                                                 protolog.addSource(OriginalSourceType.Import, String.valueOf(nameFactId), NAMESPACE, null, null, null, null);
 150                                                                 description.addElement(protolog);
 151                                                                 taxonNameBase.addDescription(description);
 152                                                                 if (citation != null){
 153                                                                         description.addSource(OriginalSourceType.PrimaryTaxonomicSource, null, null, citation, null);
 154                                                                         protolog.addSource(OriginalSourceType.PrimaryTaxonomicSource, null, null, citation, nameFactRefDetail, null, null);
 155                                                                 }
 156                                                         }//end NAME_FACT_PROTOLOGUE
 157                                                 }catch(NullPointerException e){
 158                                                         logger.warn("MediaUrl and/or MediaPath not set. Could not get protologue.");
 159                                                         success = false;
 160                                                 }
 161                                         }else if (category.equalsIgnoreCase(NAME_FACT_ALSO_PUBLISHED_IN)){
 162                                                 if (StringUtils.isNotBlank(nameFact)){
 163                                                         TaxonNameDescription description = TaxonNameDescription.NewInstance();
 164                                                         TextData additionalPublication = TextData.NewInstance(Feature.ADDITIONAL_PUBLICATION());
 165                                                         //TODO language
 166                                                         Language language = Language.DEFAULT();
 167                                                         additionalPublication.putText(language, nameFact);
 168                                                         additionalPublication.addSource(OriginalSourceType.Import, String.valueOf(nameFactId), NAMESPACE, null,null, null, null);
 169                                                         if (citation != null || isNotBlank(nameFactRefDetail)){
 170                                                                 additionalPublication.addSource(OriginalSourceType.PrimaryTaxonomicSource, null, null, citation, nameFactRefDetail, null, null);
 171                                                         }
 172                                                         description.addElement(additionalPublication);
 173                                                         taxonNameBase.addDescription(description);
 174                                                 }
 175                                         }else if (category.equalsIgnoreCase(NAME_FACT_BIBLIOGRAPHY)){
 176                                                 if (StringUtils.isNotBlank(nameFact)){
 177                                                         TaxonNameDescription description = TaxonNameDescription.NewInstance();
 178                                                         TextData bibliography = TextData.NewInstance(Feature.CITATION());
 179                                                         //TODO language
 180                                                         Language language = Language.DEFAULT();
 181                                                         bibliography.putText(language, nameFact);
 182                                                         bibliography.addSource(OriginalSourceType.Import, String.valueOf(nameFactId), NAMESPACE, null,null, null, null);
 183                                                         if (citation != null || isNotBlank(nameFactRefDetail)){
 184                                                                 bibliography.addSource(OriginalSourceType.PrimaryTaxonomicSource, null, null, citation, nameFactRefDetail, null, null);
 185                                                         }
 186                                                         description.addElement(bibliography);
 187                                                         taxonNameBase.addDescription(description);
 188                                                 }
 189                                         }else {
 190                                                 //TODO
 191                                                 logger.warn("NameFactCategory '" + category + "' not yet implemented");
 192                                                 success = false;
 193                                         }
 194
 195                                         //TODO
 196 //                                      DoubtfulFlag    bit        Checked
 197 //                                      PublishFlag      bit        Checked
 198 //                                      Created_When  datetime           Checked
 199 //                                      Updated_When datetime           Checked
 200 //                                      Created_Who    nvarchar(255)    Checked
 201 //                                      Updated_Who  nvarchar(255)    Checked
 202 //                                      Notes      nvarchar(1000)           Checked
 203
 204                                         nameToSave.add(taxonNameBase);
 205                                 }else{
 206                                         //TODO
 207                                         logger.warn("TaxonName for NameFact " + nameFactId + " does not exist in store");
 208                                         success = false;
 209                                 }
 210                                 //put
 211                         }
 212                         if (config.getMaximumNumberOfNameFacts() != 0 && i >= config.getMaximumNumberOfNameFacts() - 1){
 213                                 logger.warn("ONLY " + config.getMaximumNumberOfNameFacts() + " NAMEFACTS imported !!!" )
 214                         ;};
 215                         logger.info("Names to save: " + nameToSave.size());
 216                         getNameService().save(nameToSave);
 217                         return success;
 218                 } catch (SQLException e) {
 219                         logger.error("SQLException:" +  e);
 220                         return false;
 221                 }
 222
 223         }
 224
 225
 226         @Override
 227         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, BerlinModelImportState state) {
 228                 String nameSpace;
 229                 Class<?> cdmClass;
 230                 Set<String> idSet;
 231                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
 232
 233                 try{
 234                         Set<String> nameIdSet = new HashSet<String>();
 235                         Set<String> referenceIdSet = new HashSet<String>();
 236                         while (rs.next()){
 237                                 handleForeignKey(rs, nameIdSet, "PTnameFk");
 238                                 handleForeignKey(rs, referenceIdSet, "nameFactRefFk");
 239         }
 240
 241                         //name map
 242                         nameSpace = BerlinModelTaxonNameImport.NAMESPACE;
 243                         cdmClass = TaxonName.class;
 244                         idSet = nameIdSet;
 245                         Map<String, Person> objectMap = (Map<String, Person>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 246                         result.put(nameSpace, objectMap);
 247
 248                         //reference map
 249                         nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
 250                         cdmClass = Reference.class;
 251                         idSet = referenceIdSet;
 252                         Map<String, Reference> referenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 253                         result.put(nameSpace, referenceMap);
 254
 255
 256                 } catch (SQLException e) {
 257                         throw new RuntimeException(e);
 258                 }
 259                 return result;
 260         }
 261
 262
 263         //FIXME gibt es da keine allgemeine Methode in common?
 264         public Media getMedia(String nameFact, URL mediaUrl, File mediaPath){
 265                 if (mediaUrl == null){
 266                         logger.warn("Media Url should not be null");
 267                         return null;
 268                 }
 269                 String mimeTypeTif = "image/tiff";
 270                 String mimeTypeJpg = "image/jpeg";
 271                 String mimeTypePng = "image/png";
 272                 String mimeTypePdf = "application/pdf";
 273                 String suffixTif = "tif";
 274                 String suffixJpg = "jpg";
 275                 String suffixPng = "png";
 276                 String suffixPdf = "pdf";
 277
 278                 String sep = File.separator;
 279                 Integer size = null;
 280
 281                 logger.debug("Getting media for NameFact: " + nameFact);
 282
 283                 Media media = Media.NewInstance();
 284
 285                 String mediaUrlString = mediaUrl.toString();
 286
 287                 //tiff
 288                 String urlStringTif = mediaUrlString + "tif/" + nameFact + "." + suffixTif;
 289                 File file = new File(mediaPath, "tif" + sep + nameFact + "." + suffixTif);
 290                 MediaRepresentation representationTif = MediaRepresentation.NewInstance(mimeTypeTif, suffixTif);
 291                 if (file.exists()){
 292                         representationTif.addRepresentationPart(makeImage(urlStringTif, size, file));
 293                 }
 294                 if(representationTif.getParts().size() > 0){
 295                         media.addRepresentation(representationTif);
 296                 }
 297                 // end tif
 298                 // jpg
 299                 boolean fileExists = true;
 300                 int jpgCount = 0;
 301                 MediaRepresentation representationJpg = MediaRepresentation.NewInstance(mimeTypeJpg, suffixJpg);
 302                 while(fileExists){
 303                         String urlStringJpeg = mediaUrlString + "cmd_jpg/" + nameFact + "_page_000" + jpgCount + "." + suffixJpg;
 304                         file = new File(mediaPath, "cmd_jpg" + sep + nameFact + "_page_000" + jpgCount + "." + suffixJpg);
 305                         jpgCount++;
 306                         if (file.exists()){
 307                                 representationJpg.addRepresentationPart(makeImage(urlStringJpeg, size, file));
 308                         }else{
 309                                 fileExists = false;
 310                         }
 311                 }
 312                 if(representationJpg.getParts().size() > 0){
 313                         media.addRepresentation(representationJpg);
 314                 }
 315                 // end jpg
 316                 //png
 317                 String urlStringPng = mediaUrlString + "png/" + nameFact + "." + suffixPng;
 318                 file = new File(mediaPath, "png" + sep + nameFact + "." + suffixPng);
 319                 MediaRepresentation representationPng = MediaRepresentation.NewInstance(mimeTypePng, suffixPng);
 320                 if (file.exists()){
 321                         representationPng.addRepresentationPart(makeImage(urlStringPng, size, file));
 322                 }else{
 323                         fileExists = true;
 324                         int pngCount = 0;
 325                         while (fileExists){
 326                                 pngCount++;
 327                                 urlStringPng = mediaUrlString + "png/" + nameFact + "00" + pngCount + "." + suffixPng;
 328                                 file = new File(mediaPath, "png" + sep + nameFact + "00" + pngCount + "." + suffixPng);
 329
 330                                 if (file.exists()){
 331                                         representationPng.addRepresentationPart(makeImage(urlStringPng, size, file));
 332                                 }else{
 333                                         fileExists = false;
 334                                 }
 335                         }
 336                 }
 337                 if(representationPng.getParts().size() > 0){
 338                         media.addRepresentation(representationPng);
 339                 }
 340                 //end png
 341         //pdf
 342         String urlStringPdf = mediaUrlString + "pdf/" + nameFact + "." + suffixPdf;
 343         URI uriPdf;
 344                 try {
 345                         uriPdf = new URI(urlStringPdf);
 346                         file = new File(mediaPath, "pdf" + sep + nameFact + "." + suffixPdf);
 347                 MediaRepresentation representationPdf = MediaRepresentation.NewInstance(mimeTypePdf, suffixPdf);
 348                 if (file.exists()){
 349                         representationPdf.addRepresentationPart(MediaRepresentationPart.NewInstance(uriPdf, size));
 350                 }else{
 351                         fileExists = true;
 352                         int pdfCount = 0;
 353                         while (fileExists){
 354                                 pdfCount++;
 355                                 urlStringPdf = mediaUrlString + "pdf/" + nameFact + "00" + pdfCount + "." + suffixPdf;
 356                                 file = new File(mediaPath, "pdf/" + sep + nameFact + "00" + pdfCount + "." + suffixPdf);
 357
 358                                 if (file.exists()){
 359                                         representationPdf.addRepresentationPart(MediaRepresentationPart.NewInstance(uriPdf, size));
 360                                 }else{
 361                                         fileExists = false;
 362                                 }
 363                         }
 364                 }
 365                         if(representationPdf.getParts().size() > 0){
 366                         media.addRepresentation(representationPdf);
 367                 }
 368                 } catch (URISyntaxException e) {
 369                         e.printStackTrace();
 370                         logger.error("URISyntaxException" + urlStringPdf);
 371                 }
 372         //end pdf
 373
 374                 if(logger.isDebugEnabled()){
 375                         for (MediaRepresentation rep : media.getRepresentations()){
 376                                 for (MediaRepresentationPart part : rep.getParts()){
 377                                         logger.debug("in representation: " + part.getUri());
 378                                 }
 379                         }
 380                 }
 381
 382                 return media;
 383         }
 384
 385
 386         private ImageFile makeImage(String imageUri, Integer size, File file){
 387                 ImageInfo imageMetaData = null;
 388                 URI uri;
 389                 try {
 390                         uri = new URI(imageUri);
 391                         try {
 392                                 imageMetaData = ImageInfo.NewInstance(uri, 0);
 393                         } catch (IOException e) {
 394                                 logger.error("IOError reading image metadata." , e);
 395                         } catch (HttpException e) {
 396                                 logger.error("HttpException reading image metadata." , e);
 397                         }
 398                         ImageFile image = ImageFile.NewInstance(uri, size, imageMetaData);
 399                         return image;
 400                 } catch (URISyntaxException e1) {
 401                         logger.warn("URISyntaxException: " + imageUri);
 402                         return null;
 403                 }
 404
 405         }
 406
 407
 408         /* (non-Javadoc)
 409          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 410          */
 411         @Override
 412         protected boolean doCheck(BerlinModelImportState state){
 413                 IOValidator<BerlinModelImportState> validator = new BerlinModelNameFactsImportValidator();
 414                 return validator.validate(state);
 415         }
 416
 417
 418         /* (non-Javadoc)
 419          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 420          */
 421         @Override
 422     protected boolean isIgnore(BerlinModelImportState state){
 423                 return ! state.getConfig().isDoNameFacts();
 424         }
 425
 426
 427
 428         //for testing only
 429         public static void main(String[] args) {
 430
 431                 BerlinModelNameFactsImport nf = new BerlinModelNameFactsImport();
 432
 433                 URL url;
 434                 try {
 435                         url = new URL("http://wp5.e-taxonomy.eu/dataportal/cichorieae/media/protolog/");
 436                         File path = new File("/Volumes/protolog/protolog/");
 437                         if(path.exists()){
 438                                 String fact = "Acanthocephalus_amplexifolius";
 439                                 // make getMedia public for this to work
 440                                 Media media = nf.getMedia(fact, url, path);
 441                                 logger.info(media);
 442                                 for (MediaRepresentation rep : media.getRepresentations()){
 443                                         logger.info(rep.getMimeType());
 444                                         for (MediaRepresentationPart part : rep.getParts()){
 445                                                 logger.info(part.getUri());
 446                                         }
 447                                 }
 448                         }
 449                 } catch (MalformedURLException e) {
 450                         e.printStackTrace();
 451                 }
 452         }
 453 }