cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.common;
  11
  12 import java.net.MalformedURLException;
  13 import java.net.URI;
  14 import java.net.URISyntaxException;
  15 import java.sql.ResultSet;
  16 import java.sql.SQLException;
  17 import java.util.Arrays;
  18 import java.util.HashSet;
  19 import java.util.List;
  20 import java.util.Set;
  21 import java.util.UUID;
  22
  23 import org.apache.commons.lang.StringUtils;
  24 import org.apache.log4j.Logger;
  25
  26 import eu.etaxonomy.cdm.api.service.pager.Pager;
  27 import eu.etaxonomy.cdm.common.CdmUtils;
  28 import eu.etaxonomy.cdm.common.media.ImageInfo;
  29 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  30 import eu.etaxonomy.cdm.io.common.mapping.IInputTransformer;
  31 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  32 import eu.etaxonomy.cdm.io.markup.MarkupTransformer;
  33 import eu.etaxonomy.cdm.model.common.AnnotationType;
  34 import eu.etaxonomy.cdm.model.common.CdmBase;
  35 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  36 import eu.etaxonomy.cdm.model.common.DescriptionElementSource;
  37 import eu.etaxonomy.cdm.model.common.ExtensionType;
  38 import eu.etaxonomy.cdm.model.common.Figure;
  39 import eu.etaxonomy.cdm.model.common.IOriginalSource;
  40 import eu.etaxonomy.cdm.model.common.ISourceable;
  41 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  42 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  43 import eu.etaxonomy.cdm.model.common.Language;
  44 import eu.etaxonomy.cdm.model.common.Marker;
  45 import eu.etaxonomy.cdm.model.common.MarkerType;
  46 import eu.etaxonomy.cdm.model.common.OrderedTermVocabulary;
  47 import eu.etaxonomy.cdm.model.common.Representation;
  48 import eu.etaxonomy.cdm.model.common.TermVocabulary;
  49 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  50 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  51 import eu.etaxonomy.cdm.model.description.Feature;
  52 import eu.etaxonomy.cdm.model.description.PresenceTerm;
  53 import eu.etaxonomy.cdm.model.description.State;
  54 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  55 import eu.etaxonomy.cdm.model.description.TextData;
  56 import eu.etaxonomy.cdm.model.location.NamedArea;
  57 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  58 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  59 import eu.etaxonomy.cdm.model.location.ReferenceSystem;
  60 import eu.etaxonomy.cdm.model.media.ImageFile;
  61 import eu.etaxonomy.cdm.model.media.Media;
  62 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
  63 import eu.etaxonomy.cdm.model.name.NonViralName;
  64 import eu.etaxonomy.cdm.model.name.Rank;
  65 import eu.etaxonomy.cdm.model.reference.Reference;
  66 import eu.etaxonomy.cdm.model.taxon.Classification;
  67 import eu.etaxonomy.cdm.model.taxon.Synonym;
  68 import eu.etaxonomy.cdm.model.taxon.Taxon;
  69 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  70 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  71
  72 /**
  73  * @author a.mueller
  74  * @created 01.07.2008
  75  * @version 1.0
  76  */
  77 public abstract class CdmImportBase<CONFIG extends IImportConfigurator, STATE extends ImportStateBase> extends CdmIoBase<STATE> implements ICdmImport<CONFIG, STATE>{
  78         private static Logger logger = Logger.getLogger(CdmImportBase.class);
  79
  80         protected static final boolean CREATE = true;
  81         protected static final boolean IMAGE_GALLERY = true;
  82         protected static final boolean READ_MEDIA_DATA = true;
  83
  84         public static final UUID uuidUserDefinedNamedAreaLevelVocabulary = UUID.fromString("255144da-8d95-457e-a327-9752a8f85e5a");
  85         public static final UUID uuidUserDefinedNamedAreaVocabulary = UUID.fromString("b2238399-a3af-4f6d-b7eb-ff5d0899bf1b");
  86         public static final UUID uuidUserDefinedExtensionTypeVocabulary = UUID.fromString("e28c1394-1be8-4847-8b81-ab44eb6d5bc8");
  87         public static final UUID uuidUserDefinedReferenceSystemVocabulary = UUID.fromString("467591a3-10b4-4bf1-9239-f06ece33e90a");
  88         public static final UUID uuidUserDefinedFeatureVocabulary = UUID.fromString("fe5fccb3-a2f2-4b97-b199-6e2743cf1627");
  89         public static final UUID uuidUserDefinedStateVocabulary = UUID.fromString("f7cddb49-8392-4db1-8640-65b48a0e6d13");
  90         public static final UUID uuidUserDefinedTaxonRelationshipTypeVocabulary = UUID.fromString("31a324dc-408d-4877-891f-098db21744c6");
  91         public static final UUID uuidUserDefinedAnnotationTypeVocabulary = UUID.fromString("cd9ecdd2-9cae-4890-9032-ad83293ae883");
  92         public static final UUID uuidUserDefinedMarkerTypeVocabulary = UUID.fromString("5f02a261-fd7d-4fce-bbe4-21472de8cd51");
  93         public static final UUID uuidUserDefinedRankVocabulary = UUID.fromString("4dc57931-38e2-46c3-974d-413b087646ba");
  94
  95
  96         private static final String UuidOnly = "UUIDOnly";
  97         private static final String UuidLabel = "UUID or label";
  98         private static final String UuidLabelAbbrev = "UUID, label or abbreviation";
  99         private static final String UuidAbbrev = "UUID or abbreviation";
 100
 101         public enum TermMatchMode{
 102                 UUID_ONLY(0, UuidOnly)
 103                 ,UUID_LABEL(1, UuidLabel)
 104                 ,UUID_LABEL_ABBREVLABEL(2, UuidLabelAbbrev)
 105                 ,UUID_ABBREVLABEL(3, UuidAbbrev)
 106                 ;
 107
 108
 109                 private int id;
 110                 private String representation;
 111                 private TermMatchMode(int id, String representation){
 112                         this.id = id;
 113                         this.representation = representation;
 114                 }
 115                 public int getId() {
 116                         return id;
 117                 }
 118                 public String getRepresentation() {
 119                         return representation;
 120                 }
 121                 public TermMatchMode valueOf(int id){
 122                         switch (id){
 123                                 case 0: return UUID_ONLY;
 124                                 case 1: return UUID_LABEL;
 125                                 case 2: return UUID_LABEL_ABBREVLABEL;
 126                                 case 3: return UUID_ABBREVLABEL;
 127                                 default: return UUID_ONLY;
 128                         }
 129                 }
 130
 131
 132         }
 133
 134         protected Classification makeTree(STATE state, Reference reference){
 135                 String treeName = "Classification (Import)";
 136                 if (reference != null && StringUtils.isNotBlank(reference.getTitleCache())){
 137                         treeName = reference.getTitleCache();
 138                 }
 139                 Classification tree = Classification.NewInstance(treeName);
 140                 tree.setReference(reference);
 141
 142
 143                 // use defined uuid for first tree
 144                 CONFIG config = (CONFIG)state.getConfig();
 145                 if (state.countTrees() < 1 ){
 146                         tree.setUuid(config.getClassificationUuid());
 147                 }
 148                 getClassificationService().save(tree);
 149                 state.putTree(reference, tree);
 150                 return tree;
 151         }
 152
 153
 154         /**
 155          * Alternative memory saving method variant of
 156          * {@link #makeTree(STATE state, Reference ref)} which stores only the
 157          * UUID instead of the full tree in the <code>ImportStateBase</code> by
 158          * using <code>state.putTreeUuid(ref, tree);</code>
 159          *
 160          * @param state
 161          * @param ref
 162          * @return
 163          */
 164         protected Classification makeTreeMemSave(STATE state, Reference ref){
 165                 String treeName = "Classification (Import)";
 166                 if (ref != null && CdmUtils.isNotEmpty(ref.getTitleCache())){
 167                         treeName = ref.getTitleCache();
 168                 }
 169                 Classification tree = Classification.NewInstance(treeName);
 170                 tree.setReference(ref);
 171
 172
 173                 // use defined uuid for first tree
 174                 CONFIG config = (CONFIG)state.getConfig();
 175                 if (state.countTrees() < 1 ){
 176                         tree.setUuid(config.getClassificationUuid());
 177                 }
 178                 getClassificationService().save(tree);
 179                 state.putTreeUuid(ref, tree);
 180                 return tree;
 181         }
 182
 183
 184         protected ExtensionType getExtensionType(STATE state, UUID uuid, String label, String text, String labelAbbrev){
 185                 return getExtensionType(state, uuid, label, text, labelAbbrev, null);
 186         }
 187         protected ExtensionType getExtensionType(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<ExtensionType> voc){
 188                 if (uuid == null){
 189                         uuid = UUID.randomUUID();
 190                 }
 191                 ExtensionType extensionType = state.getExtensionType(uuid);
 192                 if (extensionType == null){
 193                         extensionType = (ExtensionType)getTermService().find(uuid);
 194                         if (extensionType == null){
 195                                 extensionType = ExtensionType.NewInstance(text, label, labelAbbrev);
 196                                 extensionType.setUuid(uuid);
 197                                 if (voc == null){
 198                                         boolean isOrdered = false;
 199                                         voc = getVocabulary(uuidUserDefinedExtensionTypeVocabulary, "User defined vocabulary for extension types", "User Defined Extension Types", null, null, isOrdered, extensionType);
 200                                 }
 201                                 voc.addTerm(extensionType);
 202                                 getTermService().saveOrUpdate(extensionType);
 203                         }
 204                         state.putExtensionType(extensionType);
 205                 }
 206                 return extensionType;
 207         }
 208
 209
 210         protected MarkerType getMarkerType(STATE state, String keyString) {
 211                 IInputTransformer transformer = state.getTransformer();
 212                 MarkerType markerType = null;
 213                 try {
 214                         markerType = transformer.getMarkerTypeByKey(keyString);
 215                 } catch (UndefinedTransformerMethodException e) {
 216                         logger.info("getMarkerTypeByKey not yet implemented for this import");
 217                 }
 218                 if (markerType == null ){
 219                         UUID uuid;
 220                         try {
 221                                 uuid = transformer.getMarkerTypeUuid(keyString);
 222                                 return getMarkerType(state, uuid, keyString, keyString, keyString);
 223                         } catch (UndefinedTransformerMethodException e) {
 224                                 logger.warn("getMarkerTypeUuid not yet implemented for this import");
 225                         }
 226                 }
 227                 return null;
 228         }
 229
 230         protected MarkerType getMarkerType(STATE state, UUID uuid, String label, String text, String labelAbbrev){
 231                 return getMarkerType(state, uuid, label, text, labelAbbrev, null);
 232         }
 233
 234
 235         protected MarkerType getMarkerType(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<MarkerType> voc){
 236                 if (uuid == null){
 237                         uuid = UUID.randomUUID();
 238                 }
 239                 MarkerType markerType = state.getMarkerType(uuid);
 240                 if (markerType == null){
 241                         markerType = (MarkerType)getTermService().find(uuid);
 242                         if (markerType == null){
 243                                 markerType = MarkerType.NewInstance(label, text, labelAbbrev);
 244                                 markerType.setUuid(uuid);
 245                                 if (voc == null){
 246                                         boolean isOrdered = false;
 247                                         voc = getVocabulary(uuidUserDefinedMarkerTypeVocabulary, "User defined vocabulary for marker types", "User Defined Marker Types", null, null, isOrdered, markerType);
 248                                 }
 249                                 voc.addTerm(markerType);
 250                                 getTermService().save(markerType);
 251                         }
 252                         state.putMarkerType(markerType);
 253                 }
 254                 return markerType;
 255         }
 256
 257         protected AnnotationType getAnnotationType(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<AnnotationType> voc){
 258                 if (uuid == null){
 259                         uuid = UUID.randomUUID();
 260                 }
 261                 AnnotationType annotationType = state.getAnnotationType(uuid);
 262                 if (annotationType == null){
 263                         annotationType = (AnnotationType)getTermService().find(uuid);
 264                         if (annotationType == null){
 265                                 annotationType = AnnotationType.NewInstance(label, text, labelAbbrev);
 266                                 annotationType.setUuid(uuid);
 267                                 if (voc == null){
 268                                         boolean isOrdered = false;
 269                                         voc = getVocabulary(uuidUserDefinedAnnotationTypeVocabulary, "User defined vocabulary for annotation types", "User Defined Annotation Types", null, null, isOrdered, annotationType);
 270                                 }
 271
 272                                 voc.addTerm(annotationType);
 273                                 getTermService().save(annotationType);
 274                         }
 275                         state.putAnnotationType(annotationType);
 276                 }
 277                 return annotationType;
 278         }
 279
 280
 281         protected ReferenceSystem getReferenceSystem(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary voc){
 282                 if (uuid == null){
 283                         uuid = UUID.randomUUID();
 284                 }
 285                 ReferenceSystem refSystem = state.getReferenceSystem(uuid);
 286                 if (refSystem == null){
 287                         refSystem = (ReferenceSystem)getTermService().find(uuid);
 288                         if (refSystem == null){
 289                                 refSystem = ReferenceSystem.NewInstance(text, label, labelAbbrev);
 290                                 if (voc == null){
 291                                         boolean isOrdered = false;
 292                                         voc = getVocabulary(uuidUserDefinedReferenceSystemVocabulary, "User defined vocabulary for reference systems", "User Defined Reference System", null, null, isOrdered, refSystem);
 293                                 }
 294                                 voc.addTerm(refSystem);
 295                                 refSystem.setUuid(uuid);
 296                                 getTermService().save(refSystem);
 297                         }
 298                         state.putReferenceSystem(refSystem);
 299                 }
 300                 return refSystem;
 301
 302         }
 303
 304
 305
 306         protected Rank getRank(STATE state, UUID uuid, String label, String text, String labelAbbrev,OrderedTermVocabulary<Rank> voc, Rank lowerRank){
 307                 if (uuid == null){
 308                         uuid = UUID.randomUUID();
 309                 }
 310                 Rank rank = state.getRank(uuid);
 311                 if (rank == null){
 312                         rank = (Rank)getTermService().find(uuid);
 313                         if (rank == null){
 314                                 rank = new Rank(text, label, labelAbbrev);
 315                                 if (voc == null){
 316                                         boolean isOrdered = true;
 317                                         voc = (OrderedTermVocabulary)getVocabulary(uuidUserDefinedRankVocabulary, "User defined vocabulary for ranks", "User Defined Reference System", null, null, isOrdered, rank);
 318                                 }
 319                                 if (lowerRank == null){
 320                                         voc.addTerm(rank);
 321                                 }else{
 322                                         voc.addTermAbove(rank, lowerRank);
 323                                 }
 324                                 rank.setUuid(uuid);
 325                                 getTermService().save(rank);
 326                         }
 327                         state.putRank(rank);
 328                 }
 329                 return rank;
 330
 331         }
 332
 333         /**
 334          * Returns a named area for a given uuid by first . If the named area does not
 335          * @param state
 336          * @param uuid
 337          * @param label
 338          * @param text
 339          * @param labelAbbrev
 340          * @param areaType
 341          * @param level
 342          * @return
 343          */
 344         protected NamedArea getNamedArea(STATE state, UUID uuid, String label, String text, String labelAbbrev, NamedAreaType areaType, NamedAreaLevel level){
 345                 return getNamedArea(state, uuid, label, text, labelAbbrev, areaType, level, null, null);
 346         }
 347
 348         protected NamedArea getNamedArea(STATE state, UUID uuid, String label, String text, String labelAbbrev, NamedAreaType areaType, NamedAreaLevel level, TermVocabulary voc, TermMatchMode matchMode){
 349                 Class<NamedArea> clazz = NamedArea.class;
 350                 if (uuid == null){
 351                         uuid = UUID.randomUUID();
 352                 }
 353                 if (matchMode == null){
 354                         matchMode = TermMatchMode.UUID_ONLY;
 355                 }
 356                 NamedArea namedArea = state.getNamedArea(uuid);
 357                 if (namedArea == null){
 358                         DefinedTermBase term = getTermService().find(uuid);
 359                         namedArea = CdmBase.deproxy(term,NamedArea.class);
 360                         //TODO matching still experimental
 361                         if (namedArea == null && (matchMode.equals(TermMatchMode.UUID_LABEL) || matchMode.equals(TermMatchMode.UUID_LABEL_ABBREVLABEL ))){
 362                                 //TODO test
 363                                 Pager<NamedArea> areaPager = (Pager)getTermService().findByTitle(clazz, label, null, null, null, null, null, null);
 364                                 namedArea = findBestMatchingArea(areaPager, uuid, label, text, labelAbbrev, areaType, level, voc);
 365                         }
 366                         if (namedArea == null && (matchMode.equals(TermMatchMode.UUID_ABBREVLABEL) || matchMode.equals(TermMatchMode.UUID_LABEL_ABBREVLABEL))){
 367                                 Pager<NamedArea> areaPager = getTermService().findByRepresentationAbbreviation(labelAbbrev, clazz, null, null);
 368                                 namedArea = findBestMatchingArea(areaPager, uuid, label, text, labelAbbrev, areaType, level, voc);
 369                         }
 370
 371                         if (namedArea == null){
 372                                 namedArea = NamedArea.NewInstance(text, label, labelAbbrev);
 373                                 if (voc == null){
 374                                         boolean isOrdered = true;
 375                                         voc = getVocabulary(uuidUserDefinedNamedAreaVocabulary, "User defined vocabulary for named areas", "User Defined Named Areas", null, null, isOrdered, namedArea);
 376                                 }
 377                                 voc.addTerm(namedArea);
 378                                 namedArea.setType(areaType);
 379                                 namedArea.setLevel(level);
 380                                 namedArea.setUuid(uuid);
 381                                 getTermService().save(namedArea);
 382                         }
 383                         state.putNamedArea(namedArea);
 384                 }
 385                 return namedArea;
 386         }
 387
 388
 389         private NamedArea findBestMatchingArea(Pager<NamedArea> areaPager, UUID uuid, String label, String text, String abbrev,
 390                         NamedAreaType areaType, NamedAreaLevel level, TermVocabulary voc) {
 391                 // TODO preliminary implementation
 392                 List<NamedArea> list = areaPager.getRecords();
 393                 if (list.size() == 0){
 394                         return null;
 395                 }else if (list.size() == 1){
 396                         return list.get(0);
 397                 }else if (list.size() > 1){
 398                         String message = "There is more than 1 matching area for %s, %s, %s. As a preliminary implementation I take the first";
 399                         message = String.format(message, label, abbrev, text);
 400                         logger.warn(message);
 401                         return list.get(0);
 402                 }
 403                 return null;
 404         }
 405
 406
 407         protected NamedAreaLevel getNamedAreaLevel(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<NamedAreaLevel> voc){
 408                 if (uuid == null){
 409                         uuid = UUID.randomUUID();
 410                 }
 411                 NamedAreaLevel namedAreaLevel = state.getNamedAreaLevel(uuid);
 412                 if (namedAreaLevel == null){
 413                         //TODO propPath just for testing
 414                         List<String> propPath = Arrays.asList("vocabulary");
 415                         DefinedTermBase<NamedAreaLevel> term = getTermService().load(uuid, propPath);
 416                         namedAreaLevel = CdmBase.deproxy(term, NamedAreaLevel.class);
 417                         if (namedAreaLevel == null){
 418                                 namedAreaLevel = NamedAreaLevel.NewInstance(text, label, labelAbbrev);
 419                                 if (voc == null){
 420                                         boolean isOrdered = true;
 421                                         voc = getVocabulary(uuidUserDefinedNamedAreaLevelVocabulary, "User defined vocabulary for named area levels", "User Defined Named Area Levels", null, null, isOrdered, namedAreaLevel);
 422                                 }
 423                                 //FIXME only for debugging
 424                                 Set<NamedAreaLevel> terms = voc.getTerms();
 425                                 for (NamedAreaLevel level : terms){
 426                                         TermVocabulary<NamedAreaLevel> levelVoc = level.getVocabulary();
 427                                         if (levelVoc == null){
 428                                                 logger.error("ONLY FOR DEBUG: Level voc is null");
 429                                         }else{
 430                                                 logger.warn("ONLY FOR DEBUG: Level voc is not null");
 431                                         }
 432                                 }
 433                                 voc.addTerm(namedAreaLevel);
 434                                 namedAreaLevel.setUuid(uuid);
 435                                 getTermService().save(namedAreaLevel);
 436                         }
 437                         state.putNamedAreaLevel(namedAreaLevel);
 438                 }
 439                 return namedAreaLevel;
 440         }
 441
 442         /**
 443          * Returns a {@link State} if it exists. <code>null</code> otherwise.
 444          * @param state
 445          * @param uuid
 446          * @return {@link State}
 447          */
 448         protected State getStateTerm(STATE state, UUID uuid){
 449                 return getStateTerm(state, uuid, null, null, null, null);
 450         }
 451
 452
 453         /**
 454          * Returns a {@link State} for a given uuid by first checking if the uuid has already been used in this import, if not
 455          * checking if the state exists in the database, if not creating it anew (with vocabulary etc.).
 456          * If label, text and labelAbbrev are all <code>null</code> no state is created.
 457          * @param importState
 458          * @param uuid
 459          * @param label
 460          * @param text
 461          * @param labelAbbrev
 462          * @param voc
 463          * @return
 464          */
 465         protected State getStateTerm(STATE importState, UUID uuid, String label, String text, String labelAbbrev, OrderedTermVocabulary<State> voc) {
 466                 if (uuid == null){
 467                         return null;
 468                 }
 469                 State stateTerm = importState.getStateTerm(uuid);
 470                 if (stateTerm == null){
 471                         stateTerm = CdmBase.deproxy(getTermService().find(uuid), State.class);
 472                         if (stateTerm == null && ! hasNoLabel(label, text, labelAbbrev)){
 473                                 stateTerm = State.NewInstance(text, label, labelAbbrev);
 474                                 stateTerm.setUuid(uuid);
 475                                 if (voc == null){
 476                                         boolean isOrdered = true;
 477                                         TermVocabulary<State> orderedVoc = getVocabulary(uuidUserDefinedFeatureVocabulary, "User defined vocabulary for states used by Categorical Data", "User Defined States", null, null, isOrdered, stateTerm);
 478                                         voc = CdmBase.deproxy(orderedVoc, OrderedTermVocabulary.class);
 479                                 }
 480                                 voc.addTerm(stateTerm);
 481                                 getTermService().save(stateTerm);
 482                         }else{
 483                                 logger.warn("No label provided for new state with uuid " + uuid);
 484                         }
 485                         importState.putStateTerm(stateTerm);
 486                 }
 487                 return stateTerm;
 488         }
 489
 490         /**
 491          * Returns a feature if it exists, null otherwise.
 492          * @see #getFeature(ImportStateBase, UUID, String, String, String, TermVocabulary)
 493          * @param state
 494          * @param uuid
 495          * @return
 496          */
 497         protected Feature getFeature(STATE state, UUID uuid){
 498                 return getFeature(state, uuid, null, null, null, null);
 499         }
 500
 501         /**
 502          * Returns a feature for a given uuid by first checking if the uuid has already been used in this import, if not
 503          * checking if the feature exists in the database, if not creating it anew (with vocabulary etc.).
 504          * If label, text and labelAbbrev are all <code>null</code> no feature is created.
 505          * @param state
 506          * @param uuid
 507          * @param label
 508          * @param text
 509          * @param labelAbbrev
 510          * @return
 511          */
 512         protected Feature getFeature(STATE state, UUID uuid, String label, String description, String labelAbbrev, TermVocabulary<Feature> voc){
 513                 if (uuid == null){
 514                         return null;
 515                 }
 516                 Feature feature = state.getFeature(uuid);
 517                 if (feature == null){
 518                         feature = (Feature)getTermService().find(uuid);
 519                         if (feature == null && ! hasNoLabel(label, description, labelAbbrev)){
 520                                 feature = Feature.NewInstance(description, label, labelAbbrev);
 521                                 feature.setUuid(uuid);
 522                                 feature.setSupportsTextData(true);
 523 //                              UUID uuidFeatureVoc = UUID.fromString("b187d555-f06f-4d65-9e53-da7c93f8eaa8");
 524                                 if (voc == null){
 525                                         boolean isOrdered = false;
 526                                         voc = getVocabulary(uuidUserDefinedFeatureVocabulary, "User defined vocabulary for features", "User Defined Features", null, null, isOrdered, feature);
 527                                 }
 528                                 voc.addTerm(feature);
 529                                 getTermService().save(feature);
 530                         }
 531                         state.putFeature(feature);
 532                 }
 533                 return feature;
 534         }
 535
 536         /**
 537          * Returns a taxon relationship type for a given uuid by first checking if the uuid has already been used in this import, if not
 538          * checking if the taxon relationship type exists in the database, if not creating it anew (with vocabulary etc.).
 539          * If label, text and labelAbbrev are all <code>null</code> no taxon relationship type is created.
 540          * @param state
 541          * @param uuid
 542          * @param label
 543          * @param text
 544          * @param labelAbbrev
 545          * @return
 546          */
 547         protected TaxonRelationshipType getTaxonRelationshipType(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<TaxonRelationshipType> voc){
 548                 if (uuid == null){
 549                         return null;
 550                 }
 551                 TaxonRelationshipType relType = state.getTaxonRelationshipType(uuid);
 552                 if (relType == null){
 553                         relType = (TaxonRelationshipType)getTermService().find(uuid);
 554                         if (relType == null && ! hasNoLabel(label, text, labelAbbrev)){
 555                                 relType = new TaxonRelationshipType();
 556                                 Representation repr = Representation.NewInstance(text, label, labelAbbrev, Language.DEFAULT());
 557                                 relType.addRepresentation(repr);
 558                                 relType.setUuid(uuid);
 559                                 if (voc == null){
 560                                         boolean isOrdered = true;
 561                                         voc = getVocabulary(uuidUserDefinedTaxonRelationshipTypeVocabulary, "User defined vocabulary for taxon relationship types", "User Defined Taxon Relationship Types", null, null, isOrdered, relType);
 562                                 }
 563                                 voc.addTerm(relType);
 564                                 getTermService().save(relType);
 565                         }
 566                         state.putTaxonRelationshipType(relType);
 567                 }
 568                 return relType;
 569         }
 570
 571         private boolean hasNoLabel(String label, String text, String labelAbbrev) {
 572                 return label == null && text == null && labelAbbrev == null;
 573         }
 574
 575
 576         /**
 577          * Returns a presence term for a given uuid by first ...
 578          * @param state
 579          * @param uuid
 580          * @param label
 581          * @param text
 582          * @param labelAbbrev
 583          * @return
 584          */
 585         protected PresenceTerm getPresenceTerm(STATE state, UUID uuid, String label, String text, String labelAbbrev){
 586                 if (uuid == null){
 587                         return null;
 588                 }
 589                 PresenceTerm presenceTerm = state.getPresenceTerm(uuid);
 590                 if (presenceTerm == null){
 591                         presenceTerm = (PresenceTerm)getTermService().find(uuid);
 592                         if (presenceTerm == null){
 593                                 presenceTerm = PresenceTerm.NewInstance(text, label, labelAbbrev);
 594                                 presenceTerm.setUuid(uuid);
 595                                 //set vocabulary ; FIXME use another user-defined vocabulary
 596                                 UUID uuidPresenceVoc = UUID.fromString("adbbbe15-c4d3-47b7-80a8-c7d104e53a05");
 597                                 TermVocabulary<PresenceTerm> voc = getVocabularyService().find(uuidPresenceVoc);
 598                                 voc.addTerm(presenceTerm);
 599                                 getTermService().save(presenceTerm);
 600                         }
 601                         state.putPresenceTerm(presenceTerm);
 602                 }
 603                 return presenceTerm;
 604         }
 605
 606         /**
 607          * Returns a language for a given uuid by first ...
 608          * @param state
 609          * @param uuid
 610          * @param label
 611          * @param text
 612          * @param labelAbbrev
 613          * @return
 614          */
 615         protected Language getLanguage(STATE state, UUID uuid, String label, String text, String labelAbbrev){
 616                 return getLanguage(state, uuid, label, text, labelAbbrev, null);
 617         }
 618
 619         protected Language getLanguage(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary voc){
 620                 if (uuid == null){
 621                         return null;
 622                 }
 623                 Language language = state.getLanguage(uuid);
 624                 if (language == null){
 625                         language = (Language)getTermService().find(uuid);
 626                         if (language == null){
 627                                 language = Language.NewInstance(text, label, labelAbbrev);
 628
 629                                 language.setUuid(uuid);
 630                                 if (voc == null){
 631                                         UUID uuidLanguageVoc = UUID.fromString("463a96f1-20ba-4a4c-9133-854c1682bd9b");
 632                                         boolean isOrdered = false;
 633                                         voc = getVocabulary(uuidLanguageVoc, "User defined languages", "User defined languages", "User defined languages", null, isOrdered, language);
 634                                 }
 635                                 //set vocabulary ; FIXME use another user-defined vocabulary
 636
 637                                 voc.addTerm(language);
 638                                 getTermService().save(language);
 639                         }
 640                         state.putLanguage(language);
 641                 }
 642                 return language;
 643         }
 644
 645
 646         /**
 647          * @param uuid
 648          * @return
 649          *
 650          */
 651         protected <T extends DefinedTermBase> TermVocabulary<T> getVocabulary(UUID uuid, String text, String label, String abbrev, URI termSourceUri, boolean isOrdered, T type) {
 652                 List<String> propPath = Arrays.asList(new String[]{"terms"});
 653                 TermVocabulary<T> voc = getVocabularyService().load(uuid, propPath);
 654                 if (voc == null){
 655                         if (isOrdered){
 656                                 voc = OrderedTermVocabulary.NewInstance(text, label, abbrev, termSourceUri);
 657                         }else{
 658                                 voc = TermVocabulary.NewInstance(text, label, abbrev, termSourceUri);
 659                         }
 660                         voc.setUuid(uuid);
 661                         getVocabularyService().save(voc);
 662                 }
 663                 return voc;
 664         }
 665
 666         /**
 667          * Adds an orginal source to a sourceable objects (implemented for Identifiable entity and description element.
 668          * If cdmBase is not sourceable nothing happens.
 669          * TODO Move to DbImportBase once this exists.
 670          * TODO also implemented in DbImportObjectCreationMapper (reduce redundance)
 671          * @param rs
 672          * @param cdmBase
 673          * @param dbIdAttribute
 674          * @param namespace
 675          * @param citation
 676          * @throws SQLException
 677          */
 678         public void addOriginalSource(CdmBase cdmBase, Object idAttributeValue, String namespace, Reference citation)  {
 679                 if (cdmBase instanceof ISourceable ){
 680                         IOriginalSource source;
 681                         ISourceable sourceable = (ISourceable)cdmBase;
 682                         Object id = idAttributeValue;
 683                         String strId = String.valueOf(id);
 684                         String microCitation = null;
 685                         if (cdmBase instanceof IdentifiableEntity){
 686                                 source = IdentifiableSource.NewInstance(strId, namespace, citation, microCitation);
 687                         }else if (cdmBase instanceof DescriptionElementBase){
 688                                 source = DescriptionElementSource.NewInstance(strId, namespace, citation, microCitation);
 689                         }else{
 690                                 logger.warn("ISourceable not beeing identifiable entities or description element base are not yet supported. CdmBase is of type " + cdmBase.getClass().getName() + ". Original source not added.");
 691                                 return;
 692                         }
 693                         sourceable.addSource(source);
 694                 }else if (cdmBase != null){
 695                         logger.warn("Sourced object does not implement ISourceable: " + cdmBase.getClass() + "," + cdmBase.getUuid());
 696                 }else{
 697                         logger.warn("Sourced object is null");
 698                 }
 699         }
 700
 701         /**
 702          * @see #addOriginalSource(CdmBase, Object, String, Reference)
 703          * @param rs
 704          * @param cdmBase
 705          * @param dbIdAttribute
 706          * @param namespace
 707          * @param citation
 708          * @throws SQLException
 709          */
 710         public void addOriginalSource(ResultSet rs, CdmBase cdmBase, String dbIdAttribute, String namespace, Reference citation) throws SQLException {
 711                 Object id = rs.getObject(dbIdAttribute);
 712                 addOriginalSource(cdmBase, id, namespace, citation);
 713         }
 714
 715
 716         /**
 717          * If the child taxon is missing genus or species epithet information and the rank is below <i>genus</i>
 718          * or <i>species</i> respectively the according epithets are taken from the parent taxon.
 719          * If the name is an autonym and has no combination author/basionym author the authors are taken from
 720          * the parent.
 721          * @param parentTaxon
 722          * @param childTaxon
 723          */
 724         protected void fillMissingEpithetsForTaxa(Taxon parentTaxon, Taxon childTaxon) {
 725                 NonViralName parentName = HibernateProxyHelper.deproxy(parentTaxon.getName(), NonViralName.class);
 726                 NonViralName childName = HibernateProxyHelper.deproxy(childTaxon.getName(), NonViralName.class);
 727                 fillMissingEpithets(parentName, childName);
 728         }
 729
 730         /**
 731          * If the child name is missing genus or species epithet information and the rank is below <i>genus</i>
 732          * or <i>species</i> respectively the according epithets are taken from the parent name.
 733          * If the name is an autonym and has no combination author/basionym author the authors are taken from
 734          * the parent.
 735          * @param parentTaxon
 736          * @param childTaxon
 737          */
 738         protected void fillMissingEpithets(NonViralName parentName, NonViralName childName) {
 739                 if (CdmUtils.isEmpty(childName.getGenusOrUninomial()) && childName.getRank().isLower(Rank.GENUS()) ){
 740                         childName.setGenusOrUninomial(parentName.getGenusOrUninomial());
 741                 }
 742
 743                 if (CdmUtils.isEmpty(childName.getSpecificEpithet()) && childName.getRank().isLower(Rank.SPECIES()) ){
 744                         childName.setSpecificEpithet(parentName.getSpecificEpithet());
 745                 }
 746                 if (childName.isAutonym() && childName.getCombinationAuthorTeam() == null && childName.getBasionymAuthorTeam() == null ){
 747                         childName.setCombinationAuthorTeam(parentName.getCombinationAuthorTeam());
 748                         childName.setBasionymAuthorTeam(parentName.getBasionymAuthorTeam());
 749                 }
 750         }
 751
 752         /**
 753          * Returns the taxon description for a taxon. If there are multiple taxon descriptions
 754          * an arbitrary one is chosen.
 755          * If no taxon description exists, a new one is created if <code>createNewIfNotExists</code>
 756          * is <code>true</code>.
 757          * @param createNewIfNotExists
 758          * @param isImageGallery if true only taxon description being image galleries are considered.
 759          * If false only taxon description being no image galleries are considered.
 760          * @return
 761          */
 762         public TaxonDescription getTaxonDescription(Taxon taxon, boolean isImageGallery, boolean createNewIfNotExists) {
 763                 Reference ref = null;
 764                 return getTaxonDescription(taxon, ref, isImageGallery, createNewIfNotExists);
 765         }
 766
 767         /**
 768          * Like {@link #getTaxonDescription(Taxon, boolean, boolean)}
 769          * Only matches a description if the given reference is a source of the description.<BR>
 770          * If a new description is created the given reference will be added as a source.
 771          *
 772          * @see #getTaxonDescription(Taxon, boolean, boolean)
 773          */
 774         public TaxonDescription getTaxonDescription(Taxon taxon, Reference ref, boolean isImageGallery, boolean createNewIfNotExists) {
 775                 TaxonDescription result = null;
 776                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
 777                 for (TaxonDescription description : descriptions){
 778                         if (description.isImageGallery() == isImageGallery){
 779                                 if (hasCorrespondingSource(ref, description)){
 780                                         result = description;
 781                                         break;
 782                                 }
 783                         }
 784                 }
 785                 if (result == null && createNewIfNotExists){
 786                         result = TaxonDescription.NewInstance(taxon);
 787                         result.setImageGallery(isImageGallery);
 788                         if (ref != null){
 789                                 result.addSource(null, null, ref, null);
 790                         }
 791                 }
 792                 return result;
 793         }
 794
 795
 796         /**
 797          * Returns the textdata that holds general information about a feature for a taxon description.
 798          * This is mainly necessary for descriptions that have more than one description element for
 799          * a given feature such as 'distribution', 'description' or 'common name'. It may also hold
 800          * for hierarchical features where no description element exists for a higher hierarchie level.
 801          * Example: the description feature has subfeatures. But some information like authorship, figures,
 802          * sources need to be added to the description itself.
 803          * Currently a feature placeholder is marked by a marker of type 'feature placeholder'. Maybe in future
 804          * there will be a boolean marker in the TextData class itself.
 805          * @param state
 806          * @param feature
 807          * @param taxon
 808          * @param ref
 809          * @param createIfNotExists
 810          * @return
 811          */
 812         protected TextData getFeaturePlaceholder(STATE state, DescriptionBase<?> description, Feature feature, boolean createIfNotExists) {
 813                 UUID featurePlaceholderUuid = MarkupTransformer.uuidFeaturePlaceholder;
 814                 for (DescriptionElementBase element : description.getElements()){
 815                         if (element.isInstanceOf(TextData.class)){
 816                                 TextData textData = CdmBase.deproxy(element, TextData.class);
 817                                 if (textData.getFeature() == null || ! textData.getFeature().equals(feature)){
 818                                         continue;
 819                                 }
 820                                 for (Marker marker : textData.getMarkers()){
 821                                         MarkerType markerType = marker.getMarkerType();
 822                                         if (markerType != null &&
 823                                                         markerType.getUuid().equals(featurePlaceholderUuid) &&
 824                                                         marker.getValue() == true){
 825                                                 return textData;
 826                                         }
 827                                 }
 828                         }
 829                 }
 830                 if (createIfNotExists){
 831                         TextData newPlaceholder = TextData.NewInstance(feature);
 832                         MarkerType placeholderMarkerType = getMarkerType(state, featurePlaceholderUuid, "Feature Placeholder", "Feature Placeholder", null);
 833                         Marker marker = Marker.NewInstance(placeholderMarkerType, true);
 834                         newPlaceholder.addMarker(marker);
 835                         description.addElement(newPlaceholder);
 836                         return newPlaceholder;
 837                 }else{
 838                         return null;
 839                 }
 840         }
 841
 842
 843
 844         /**
 845          * Returns true, if this description has a source with a citation equal to the given reference.
 846          * Returns true if the given reference is null.
 847          * @param ref
 848          * @param description
 849          */
 850         private boolean hasCorrespondingSource(Reference ref, TaxonDescription description) {
 851                 if (ref != null){
 852                         for (IdentifiableSource source : description.getSources()){
 853                                 if (ref.equals(source.getCitation())){
 854                                         return true;
 855                                 }
 856                         }
 857                         return false;
 858                 }
 859                 return true;
 860
 861         }
 862
 863
 864         /**
 865          * Returns the accepted taxon of a {@link TaxonBase taxon base}. <BR>
 866          * If taxonBase is of type taxon the same object is returned. If taxonBase is of type
 867          * synonym the accepted taxon is returned if one exists. If no accepted taxon exists
 868          * <code>null</code> is returned. If multiple accepted taxa exist the one taxon with the
 869          * same secundum reference is returned. If no such single taxon exists an
 870          * {@link IllegalStateException illegal state exception} is thrown.
 871          * @param taxonBase
 872          * @return
 873          */
 874         protected Taxon getAcceptedTaxon(TaxonBase<?> taxonBase) {
 875                 if (taxonBase == null){
 876                         return null;
 877                 }else if(taxonBase.isInstanceOf(Taxon.class)){
 878                         return CdmBase.deproxy(taxonBase, Taxon.class);
 879                 }else if(taxonBase.isInstanceOf(Synonym.class)){
 880                         Synonym synonym = CdmBase.deproxy(taxonBase, Synonym.class);
 881                         Set<Taxon> acceptedTaxa = synonym.getAcceptedTaxa();
 882                         if (acceptedTaxa.size() == 0){
 883                                 return null;
 884                         }else if (acceptedTaxa.size() == 1){
 885                                 return acceptedTaxa.iterator().next();
 886                         }else{
 887                                 Reference sec = synonym.getSec();
 888                                 if (sec != null){
 889                                         Set<Taxon> taxaWithSameSec = new HashSet<Taxon>();
 890                                         for (Taxon taxon: acceptedTaxa){
 891                                                 if (sec.equals(taxon.getSec())){
 892                                                         taxaWithSameSec.add(taxon);
 893                                                 }
 894                                         }
 895                                         if (taxaWithSameSec.size() == 1){
 896                                                 return taxaWithSameSec.iterator().next();
 897                                         }
 898                                 }
 899                                 throw new IllegalStateException("Can't define the one accepted taxon for a synonym out of multiple accept taxa");
 900                         }
 901                 }else{
 902                         throw new IllegalStateException("Unknown TaxonBase subclass: " + taxonBase.getClass().getName());
 903                 }
 904         }
 905
 906
 907
 908         /**
 909          * Creates
 910          * @param uriString
 911          * @param readDataFromUrl
 912          * @see #READ_MEDIA_DATA
 913          * @return
 914          * @throws MalformedURLException
 915          */
 916         protected Media getImageMedia(String uriString, boolean readMediaData, boolean isFigure) throws MalformedURLException {
 917                 if( uriString == null){
 918                         return null;
 919                 } else {
 920                         ImageInfo imageInfo = null;
 921                         URI uri;
 922                         try {
 923                                 uri = new URI(uriString);
 924                                 try {
 925                                         if (readMediaData){
 926                                                 imageInfo = ImageInfo.NewInstance(uri, 0);
 927                                         }
 928                                 } catch (Exception e) {
 929                                         String message = "An error occurred when trying to read image meta data: " +  e.getMessage();
 930                                         logger.warn(message);
 931                                         fireWarningEvent(message, "unknown location", 2, 0);
 932                                 }
 933                                 ImageFile imageFile = ImageFile.NewInstance(uri, null, imageInfo);
 934                                 MediaRepresentation representation = MediaRepresentation.NewInstance();
 935                                 if(imageInfo != null){
 936                                         representation.setMimeType(imageInfo.getMimeType());
 937                                         representation.setSuffix(imageInfo.getSuffix());
 938                                 }
 939                                 representation.addRepresentationPart(imageFile);
 940                                 Media media = isFigure ? Figure.NewInstance() : Media.NewInstance();
 941                                 media.addRepresentation(representation);
 942                                 return media;
 943                         } catch (URISyntaxException e1) {
 944                                 String message = "An URISyntaxException occurred when trying to create uri from multimedia objcet string: " +  uriString;
 945                                 logger.warn(message);
 946                                 fireWarningEvent(message, "unknown location", 4, 0);
 947                                 return null;
 948                         }
 949                 }
 950         }
 951
 952
 953         /**
 954          * Retrieves an Integer value from a result set. If the value is NULL null is returned.
 955          * ResultSet.getInt() returns 0 therefore we need a special handling for this case.
 956          * @param rs
 957          * @param columnName
 958          * @return
 959          * @throws SQLException
 960          */
 961         protected Integer nullSafeInt(ResultSet rs, String columnName) throws SQLException {
 962                 Object intObject = rs.getObject(columnName);
 963                 if (intObject == null){
 964                         return null;
 965                 }else{
 966                         return Integer.valueOf(intObject.toString());
 967                 }
 968         }
 969
 970         protected Double nullSafeDouble(ResultSet rs, String columnName) throws SQLException {
 971                 Object doubleObject = rs.getObject(columnName);
 972                 if (doubleObject == null){
 973                         return null;
 974                 }else{
 975                         return Double.valueOf(doubleObject.toString());
 976                 }
 977         }
 978
 979
 980 }