cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.common;
  11
  12 import java.net.MalformedURLException;
  13 import java.net.URI;
  14 import java.net.URISyntaxException;
  15 import java.sql.ResultSet;
  16 import java.sql.SQLException;
  17 import java.util.ArrayList;
  18 import java.util.Arrays;
  19 import java.util.HashSet;
  20 import java.util.List;
  21 import java.util.Set;
  22 import java.util.UUID;
  23
  24 import org.apache.commons.lang.StringUtils;
  25 import org.apache.log4j.Logger;
  26
  27 import eu.etaxonomy.cdm.api.service.pager.Pager;
  28 import eu.etaxonomy.cdm.common.media.ImageInfo;
  29 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  30 import eu.etaxonomy.cdm.io.common.mapping.IInputTransformer;
  31 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  32 import eu.etaxonomy.cdm.io.markup.MarkupTransformer;
  33 import eu.etaxonomy.cdm.model.common.AnnotationType;
  34 import eu.etaxonomy.cdm.model.common.CdmBase;
  35 import eu.etaxonomy.cdm.model.common.DefinedTerm;
  36 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  37 import eu.etaxonomy.cdm.model.common.ExtensionType;
  38 import eu.etaxonomy.cdm.model.common.IOriginalSource;
  39 import eu.etaxonomy.cdm.model.common.ISourceable;
  40 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  41 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  42 import eu.etaxonomy.cdm.model.common.Language;
  43 import eu.etaxonomy.cdm.model.common.Marker;
  44 import eu.etaxonomy.cdm.model.common.MarkerType;
  45 import eu.etaxonomy.cdm.model.common.OrderedTermVocabulary;
  46 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  47 import eu.etaxonomy.cdm.model.common.TermType;
  48 import eu.etaxonomy.cdm.model.common.TermVocabulary;
  49 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  50 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  51 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  52 import eu.etaxonomy.cdm.model.description.Feature;
  53 import eu.etaxonomy.cdm.model.description.MeasurementUnit;
  54 import eu.etaxonomy.cdm.model.description.PresenceTerm;
  55 import eu.etaxonomy.cdm.model.description.SpecimenDescription;
  56 import eu.etaxonomy.cdm.model.description.State;
  57 import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
  58 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  59 import eu.etaxonomy.cdm.model.description.TaxonNameDescription;
  60 import eu.etaxonomy.cdm.model.description.TextData;
  61 import eu.etaxonomy.cdm.model.location.Country;
  62 import eu.etaxonomy.cdm.model.location.NamedArea;
  63 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  64 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  65 import eu.etaxonomy.cdm.model.location.ReferenceSystem;
  66 import eu.etaxonomy.cdm.model.media.ImageFile;
  67 import eu.etaxonomy.cdm.model.media.Media;
  68 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
  69 import eu.etaxonomy.cdm.model.name.NonViralName;
  70 import eu.etaxonomy.cdm.model.name.Rank;
  71 import eu.etaxonomy.cdm.model.name.RankClass;
  72 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  73 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  74 import eu.etaxonomy.cdm.model.reference.Reference;
  75 import eu.etaxonomy.cdm.model.taxon.Classification;
  76 import eu.etaxonomy.cdm.model.taxon.Synonym;
  77 import eu.etaxonomy.cdm.model.taxon.Taxon;
  78 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  79 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  80
  81 /**
  82  * @author a.mueller
  83  * @created 01.07.2008
  84  * @version 1.0
  85  */
  86 public abstract class CdmImportBase<CONFIG extends IImportConfigurator, STATE extends ImportStateBase> extends CdmIoBase<STATE> implements ICdmImport<CONFIG, STATE>{
  87         private static Logger logger = Logger.getLogger(CdmImportBase.class);
  88
  89         protected static final boolean CREATE = true;
  90         protected static final boolean IMAGE_GALLERY = true;
  91         protected static final boolean READ_MEDIA_DATA = true;
  92
  93         public static final UUID uuidUserDefinedNamedAreaLevelVocabulary = UUID.fromString("255144da-8d95-457e-a327-9752a8f85e5a");
  94         public static final UUID uuidUserDefinedNamedAreaVocabulary = UUID.fromString("b2238399-a3af-4f6d-b7eb-ff5d0899bf1b");
  95         public static final UUID uuidUserDefinedExtensionTypeVocabulary = UUID.fromString("e28c1394-1be8-4847-8b81-ab44eb6d5bc8");
  96         public static final UUID uuidUserDefinedReferenceSystemVocabulary = UUID.fromString("467591a3-10b4-4bf1-9239-f06ece33e90a");
  97         public static final UUID uuidUserDefinedFeatureVocabulary = UUID.fromString("fe5fccb3-a2f2-4b97-b199-6e2743cf1627");
  98         public static final UUID uuidUserDefinedMeasurementUnitVocabulary = UUID.fromString("d5e72bb7-f312-4080-bb86-c695d04a6e66");
  99         public static final UUID uuidUserDefinedStatisticalMeasureVocabulary = UUID.fromString("62a89836-c730-4b4f-a904-3d859dbfc400");
 100         public static final UUID uuidUserDefinedStateVocabulary = UUID.fromString("f7cddb49-8392-4db1-8640-65b48a0e6d13");
 101         public static final UUID uuidUserDefinedTaxonRelationshipTypeVocabulary = UUID.fromString("31a324dc-408d-4877-891f-098db21744c6");
 102         public static final UUID uuidUserDefinedAnnotationTypeVocabulary = UUID.fromString("cd9ecdd2-9cae-4890-9032-ad83293ae883");
 103         public static final UUID uuidUserDefinedMarkerTypeVocabulary = UUID.fromString("5f02a261-fd7d-4fce-bbe4-21472de8cd51");
 104         public static final UUID uuidUserDefinedRankVocabulary = UUID.fromString("4dc57931-38e2-46c3-974d-413b087646ba");
 105
 106         public static final UUID uuidUserDefinedModifierVocabulary = UUID.fromString("2a8b3838-3a95-49ea-9ab2-3049614b5884");
 107         public static final UUID uuidUserDefinedKindOfUnitVocabulary = UUID.fromString("e7c5deb2-f485-4a66-9104-0c5398efd481");
 108
 109
 110
 111         private static final String UuidOnly = "UUIDOnly";
 112         private static final String UuidLabel = "UUID or label";
 113         private static final String UuidLabelAbbrev = "UUID, label or abbreviation";
 114         private static final String UuidAbbrev = "UUID or abbreviation";
 115
 116         public enum TermMatchMode{
 117                 UUID_ONLY(0, UuidOnly)
 118                 ,UUID_LABEL(1, UuidLabel)
 119                 ,UUID_LABEL_ABBREVLABEL(2, UuidLabelAbbrev)
 120                 ,UUID_ABBREVLABEL(3, UuidAbbrev)
 121                 ;
 122
 123
 124                 private int id;
 125                 private String representation;
 126                 private TermMatchMode(int id, String representation){
 127                         this.id = id;
 128                         this.representation = representation;
 129                 }
 130                 public int getId() {
 131                         return id;
 132                 }
 133                 public String getRepresentation() {
 134                         return representation;
 135                 }
 136                 public TermMatchMode valueOf(int id){
 137                         switch (id){
 138                                 case 0: return UUID_ONLY;
 139                                 case 1: return UUID_LABEL;
 140                                 case 2: return UUID_LABEL_ABBREVLABEL;
 141                                 case 3: return UUID_ABBREVLABEL;
 142                                 default: return UUID_ONLY;
 143                         }
 144                 }
 145
 146
 147         }
 148
 149         protected Classification makeTree(STATE state, Reference<?> reference){
 150                 String treeName = "Classification (Import)";
 151                 if (reference != null && StringUtils.isNotBlank(reference.getTitleCache())){
 152                         treeName = reference.getTitleCache();
 153                 }
 154                 Classification tree = Classification.NewInstance(treeName);
 155                 tree.setReference(reference);
 156
 157
 158                 // use defined uuid for first tree
 159                 CONFIG config = (CONFIG)state.getConfig();
 160                 if (state.countTrees() < 1 ){
 161                         tree.setUuid(config.getClassificationUuid());
 162                 }
 163                 getClassificationService().save(tree);
 164                 state.putTree(reference, tree);
 165                 return tree;
 166         }
 167
 168
 169         /**
 170          * Alternative memory saving method variant of
 171          * {@link #makeTree(STATE state, Reference ref)} which stores only the
 172          * UUID instead of the full tree in the <code>ImportStateBase</code> by
 173          * using <code>state.putTreeUuid(ref, tree);</code>
 174          *
 175          * @param state
 176          * @param ref
 177          * @return
 178          */
 179         protected Classification makeTreeMemSave(STATE state, Reference ref){
 180                 String treeName = "Classification (Import)";
 181                 if (ref != null && StringUtils.isNotBlank(ref.getTitleCache())){
 182                         treeName = ref.getTitleCache();
 183                 }
 184                 Classification tree = Classification.NewInstance(treeName);
 185                 tree.setReference(ref);
 186
 187
 188                 // use defined uuid for first tree
 189                 CONFIG config = (CONFIG)state.getConfig();
 190                 if (state.countTrees() < 1 ){
 191                         tree.setUuid(config.getClassificationUuid());
 192                 }
 193                 getClassificationService().save(tree);
 194                 state.putTreeUuid(ref, tree);
 195                 return tree;
 196         }
 197
 198
 199         protected ExtensionType getExtensionType(STATE state, UUID uuid, String label, String text, String labelAbbrev){
 200                 return getExtensionType(state, uuid, label, text, labelAbbrev, null);
 201         }
 202         protected ExtensionType getExtensionType(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<ExtensionType> voc){
 203                 if (uuid == null){
 204                         uuid = UUID.randomUUID();
 205                 }
 206                 ExtensionType extensionType = state.getExtensionType(uuid);
 207                 if (extensionType == null){
 208                         extensionType = (ExtensionType)getTermService().find(uuid);
 209                         if (extensionType == null){
 210                                 extensionType = ExtensionType.NewInstance(text, label, labelAbbrev);
 211                                 extensionType.setUuid(uuid);
 212                                 if (voc == null){
 213                                         boolean isOrdered = false;
 214                                         voc = getVocabulary(TermType.ExtensionType, uuidUserDefinedExtensionTypeVocabulary, "User defined vocabulary for extension types", "User Defined Extension Types", null, null, isOrdered, extensionType);
 215                                 }
 216                                 voc.addTerm(extensionType);
 217                                 getTermService().saveOrUpdate(extensionType);
 218                         }
 219                         state.putExtensionType(extensionType);
 220                 }
 221                 return extensionType;
 222         }
 223
 224
 225         protected MarkerType getMarkerType(STATE state, String keyString) {
 226                 IInputTransformer transformer = state.getTransformer();
 227                 MarkerType markerType = null;
 228                 try {
 229                         markerType = transformer.getMarkerTypeByKey(keyString);
 230                 } catch (UndefinedTransformerMethodException e) {
 231                         logger.info("getMarkerTypeByKey not yet implemented for this import");
 232                 }
 233                 if (markerType == null ){
 234                         UUID uuid;
 235                         try {
 236                                 uuid = transformer.getMarkerTypeUuid(keyString);
 237                                 return getMarkerType(state, uuid, keyString, keyString, keyString);
 238                         } catch (UndefinedTransformerMethodException e) {
 239                                 logger.warn("getMarkerTypeUuid not yet implemented for this import");
 240                         }
 241                 }
 242                 return null;
 243         }
 244
 245         protected MarkerType getMarkerType(STATE state, UUID uuid, String label, String text, String labelAbbrev){
 246                 return getMarkerType(state, uuid, label, text, labelAbbrev, null);
 247         }
 248
 249
 250         protected MarkerType getMarkerType(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<MarkerType> voc){
 251                 if (uuid == null){
 252                         uuid = UUID.randomUUID();
 253                 }
 254                 MarkerType markerType = state.getMarkerType(uuid);
 255                 if (markerType == null){
 256                         markerType = (MarkerType)getTermService().find(uuid);
 257                         if (markerType == null){
 258                                 markerType = MarkerType.NewInstance(label, text, labelAbbrev);
 259                                 markerType.setUuid(uuid);
 260                                 if (voc == null){
 261                                         boolean isOrdered = false;
 262                                         voc = getVocabulary(TermType.MarkerType, uuidUserDefinedMarkerTypeVocabulary, "User defined vocabulary for marker types", "User Defined Marker Types", null, null, isOrdered, markerType);
 263                                 }
 264                                 voc.addTerm(markerType);
 265                                 getTermService().save(markerType);
 266                         }
 267                         state.putMarkerType(markerType);
 268                 }
 269                 return markerType;
 270         }
 271
 272         protected AnnotationType getAnnotationType(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<AnnotationType> voc){
 273                 if (uuid == null){
 274                         uuid = UUID.randomUUID();
 275                 }
 276                 AnnotationType annotationType = state.getAnnotationType(uuid);
 277                 if (annotationType == null){
 278                         annotationType = (AnnotationType)getTermService().find(uuid);
 279                         if (annotationType == null){
 280                                 annotationType = AnnotationType.NewInstance(label, text, labelAbbrev);
 281                                 annotationType.setUuid(uuid);
 282                                 if (voc == null){
 283                                         boolean isOrdered = false;
 284                                         voc = getVocabulary(TermType.AnnotationType, uuidUserDefinedAnnotationTypeVocabulary, "User defined vocabulary for annotation types", "User Defined Annotation Types", null, null, isOrdered, annotationType);
 285                                 }
 286
 287                                 voc.addTerm(annotationType);
 288                                 getTermService().save(annotationType);
 289                         }
 290                         state.putAnnotationType(annotationType);
 291                 }
 292                 return annotationType;
 293         }
 294
 295
 296         protected ReferenceSystem getReferenceSystem(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary voc){
 297                 if (uuid == null){
 298                         uuid = UUID.randomUUID();
 299                 }
 300                 ReferenceSystem refSystem = state.getReferenceSystem(uuid);
 301                 if (refSystem == null){
 302                         refSystem = (ReferenceSystem)getTermService().find(uuid);
 303                         if (refSystem == null){
 304                                 refSystem = ReferenceSystem.NewInstance(text, label, labelAbbrev);
 305                                 if (voc == null){
 306                                         boolean isOrdered = false;
 307                                         voc = getVocabulary(TermType.ReferenceSystem, uuidUserDefinedReferenceSystemVocabulary, "User defined vocabulary for reference systems", "User Defined Reference System", null, null, isOrdered, refSystem);
 308                                 }
 309                                 voc.addTerm(refSystem);
 310                                 refSystem.setUuid(uuid);
 311                                 getTermService().save(refSystem);
 312                         }
 313                         state.putReferenceSystem(refSystem);
 314                 }
 315                 return refSystem;
 316
 317         }
 318
 319
 320
 321         protected Rank getRank(STATE state, UUID uuid, String label, String text, String labelAbbrev,OrderedTermVocabulary<Rank> voc, Rank lowerRank, RankClass rankClass){
 322                 if (uuid == null){
 323                         uuid = UUID.randomUUID();
 324                 }
 325                 Rank rank = state.getRank(uuid);
 326                 if (rank == null){
 327                         rank = (Rank)getTermService().find(uuid);
 328                         if (rank == null){
 329                                 rank = Rank.NewInstance(rankClass, text, label, labelAbbrev);
 330                                 if (voc == null){
 331                                         boolean isOrdered = true;
 332                                         voc = (OrderedTermVocabulary)getVocabulary(TermType.Rank, uuidUserDefinedRankVocabulary, "User defined vocabulary for ranks", "User Defined Reference System", null, null, isOrdered, rank);
 333                                 }
 334                                 if (lowerRank == null){
 335                                         voc.addTerm(rank);
 336                                 }else{
 337                                         voc.addTermAbove(rank, lowerRank);
 338                                 }
 339                                 rank.setUuid(uuid);
 340                                 getTermService().save(rank);
 341                         }
 342                         state.putRank(rank);
 343                 }
 344                 return rank;
 345
 346         }
 347
 348         /**
 349          * Returns a named area for a given uuid by first . If the named area does not
 350          * @param state
 351          * @param uuid
 352          * @param label
 353          * @param text
 354          * @param labelAbbrev
 355          * @param areaType
 356          * @param level
 357          * @return
 358          */
 359         protected NamedArea getNamedArea(STATE state, UUID uuid, String label, String text, String labelAbbrev, NamedAreaType areaType, NamedAreaLevel level){
 360                 return getNamedArea(state, uuid, label, text, labelAbbrev, areaType, level, null, null);
 361         }
 362
 363         protected NamedArea getNamedArea(STATE state, UUID uuid, String label, String text, String labelAbbrev, NamedAreaType areaType, NamedAreaLevel level, TermVocabulary voc, TermMatchMode matchMode){
 364                 return getNamedArea(state, uuid, label, text, labelAbbrev, areaType, level, voc, matchMode, null);
 365         }
 366
 367
 368         protected NamedArea getNamedArea(STATE state, UUID uuid, String label, String text, String labelAbbrev, NamedAreaType areaType, NamedAreaLevel level, TermVocabulary voc, TermMatchMode matchMode,
 369                         List<TermVocabulary<NamedArea>> vocabularyPreference){
 370                 Class<NamedArea> clazz = NamedArea.class;
 371                 if (uuid == null){
 372                         uuid = UUID.randomUUID();
 373                 }
 374                 if (matchMode == null){
 375                         matchMode = TermMatchMode.UUID_ONLY;
 376                 }
 377                 NamedArea namedArea = state.getNamedArea(uuid);
 378                 if (namedArea == null){
 379                         DefinedTermBase<?> term = getTermService().find(uuid);
 380                         namedArea = CdmBase.deproxy(term,NamedArea.class);
 381
 382                         if (vocabularyPreference == null){
 383                                 vocabularyPreference =  new ArrayList<TermVocabulary<NamedArea>>();
 384                         }
 385                         if (vocabularyPreference.isEmpty()){
 386                                 vocabularyPreference.add(Country.GERMANY().getVocabulary());
 387                                 vocabularyPreference.add(TdwgAreaProvider.getAreaByTdwgAbbreviation("GER").getVocabulary());
 388                         }
 389
 390
 391                         //TODO matching still experimental
 392                         if (namedArea == null && (matchMode.equals(TermMatchMode.UUID_LABEL) || matchMode.equals(TermMatchMode.UUID_LABEL_ABBREVLABEL ))){
 393                                 //TODO test
 394                                 Pager<NamedArea> areaPager = (Pager)getTermService().findByTitle(clazz, label, null, null, null, null, null, null);
 395                                 namedArea = findBestMatchingArea(areaPager, uuid, label, text, labelAbbrev, vocabularyPreference);
 396                         }
 397                         if (namedArea == null && (matchMode.equals(TermMatchMode.UUID_ABBREVLABEL) || matchMode.equals(TermMatchMode.UUID_LABEL_ABBREVLABEL))){
 398                                 Pager<NamedArea> areaPager = getTermService().findByRepresentationAbbreviation(labelAbbrev, clazz, null, null);
 399                                 namedArea = findBestMatchingArea(areaPager, uuid, label, text, labelAbbrev, vocabularyPreference);
 400                         }
 401
 402                         if (namedArea == null){
 403                                 namedArea = NamedArea.NewInstance(text, label, labelAbbrev);
 404                                 if (voc == null){
 405                                         boolean isOrdered = true;
 406                                         voc = getVocabulary(TermType.NamedArea, uuidUserDefinedNamedAreaVocabulary, "User defined vocabulary for named areas", "User Defined Named Areas", null, null, isOrdered, namedArea);
 407                                 }
 408                                 voc.addTerm(namedArea);
 409                                 namedArea.setType(areaType);
 410                                 namedArea.setLevel(level);
 411                                 namedArea.setUuid(uuid);
 412                                 getTermService().saveOrUpdate(namedArea);
 413                         }
 414                         state.putNamedArea(namedArea);
 415                 }
 416                 return namedArea;
 417         }
 418
 419
 420         private NamedArea findBestMatchingArea(Pager<NamedArea> areaPager, UUID uuid, String label, String text, String abbrev, List<TermVocabulary<NamedArea>> vocabularyPreference) {
 421                 // TODO preliminary implementation
 422                 List<NamedArea> list = areaPager.getRecords();
 423                 if (list.size() == 0){
 424                         return null;
 425                 }else if (list.size() == 1){
 426                         return list.get(0);
 427                 }else if (list.size() > 1){
 428                         List<NamedArea> preferredList = new ArrayList<NamedArea>();
 429                         for (TermVocabulary<NamedArea> voc: vocabularyPreference){
 430                                 for (NamedArea area : list){
 431                                         if (voc.equals(area.getVocabulary())){
 432                                                 preferredList.add(area);
 433                                         }
 434                                 }
 435                                 if (preferredList.size() > 0){
 436                                         break;
 437                                 }
 438                         }
 439                         if (preferredList.size() > 1 ){
 440                                 preferredList = getLowestLevelAreas(preferredList);
 441                         }else if (preferredList.size() == 0 ){
 442                                 preferredList = list;
 443                         }
 444                         if (preferredList.size() == 1 ){
 445                                 return preferredList.get(0);
 446                         }else if (preferredList.size() > 1 ){
 447                                 String message = "There is more than 1 matching area for %s, %s, %s. As a preliminary implementation I take the first";
 448                                 message = String.format(message, label, abbrev, text);
 449                                 logger.warn(message);
 450                                 return list.get(0);
 451                         }
 452                 }
 453                 return null;
 454         }
 455
 456
 457         private List<NamedArea> getLowestLevelAreas(List<NamedArea> preferredList) {
 458                 List<NamedArea> result = new ArrayList<NamedArea>();
 459                 for (NamedArea area : preferredList){
 460                         if (result.isEmpty()){
 461                                 result.add(area);
 462                         }else {
 463                                 int compare = compareAreaLevel(area, result.get(0));
 464                                 if (compare < 0){
 465                                         result = new ArrayList<NamedArea>();
 466                                         result.add(area);
 467                                 }else if (compare == 0){
 468                                         result.add(area);
 469                                 }else{
 470                                         //do nothing
 471                                 }
 472
 473                         }
 474                 }
 475
 476                 return result;
 477         }
 478
 479
 480         private int compareAreaLevel(NamedArea area1, NamedArea area2) {
 481                 NamedAreaLevel level1 = area1.getLevel();
 482                 NamedAreaLevel level2 = area2.getLevel();
 483                 if (level1 == null){
 484                         return (level2 == null)? 0 : 1;
 485                 }else if (level2 == null){
 486                         return -1;
 487                 }else{
 488                         return level1.compareTo(level2);
 489                 }
 490         }
 491
 492
 493         protected NamedAreaLevel getNamedAreaLevel(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<NamedAreaLevel> voc){
 494                 if (uuid == null){
 495                         uuid = UUID.randomUUID();
 496                 }
 497                 NamedAreaLevel namedAreaLevel = state.getNamedAreaLevel(uuid);
 498                 if (namedAreaLevel == null){
 499                         //TODO propPath just for testing
 500                         List<String> propPath = Arrays.asList("vocabulary");
 501                         DefinedTermBase<NamedAreaLevel> term = getTermService().load(uuid, propPath);
 502                         namedAreaLevel = CdmBase.deproxy(term, NamedAreaLevel.class);
 503                         if (namedAreaLevel == null){
 504                                 namedAreaLevel = NamedAreaLevel.NewInstance(text, label, labelAbbrev);
 505                                 if (voc == null){
 506                                         boolean isOrdered = true;
 507                                         voc = getVocabulary(TermType.NamedAreaLevel, uuidUserDefinedNamedAreaLevelVocabulary, "User defined vocabulary for named area levels", "User Defined Named Area Levels", null, null, isOrdered, namedAreaLevel);
 508                                 }
 509                                 //FIXME only for debugging
 510                                 Set<NamedAreaLevel> terms = voc.getTerms();
 511                                 for (NamedAreaLevel level : terms){
 512                                         TermVocabulary<NamedAreaLevel> levelVoc = level.getVocabulary();
 513                                         if (levelVoc == null){
 514                                                 logger.error("ONLY FOR DEBUG: Level voc is null");
 515                                         }else{
 516                                                 logger.info("ONLY FOR DEBUG: Level voc is not null");
 517                                         }
 518                                 }
 519                                 voc.addTerm(namedAreaLevel);
 520                                 namedAreaLevel.setUuid(uuid);
 521                                 getTermService().save(namedAreaLevel);
 522                         }
 523                         state.putNamedAreaLevel(namedAreaLevel);
 524                 }
 525                 return namedAreaLevel;
 526         }
 527
 528         /**
 529          * Returns a {@link State} if it exists. <code>null</code> otherwise.
 530          * @param state
 531          * @param uuid
 532          * @return {@link State}
 533          */
 534         protected State getStateTerm(STATE state, UUID uuid){
 535                 return getStateTerm(state, uuid, null, null, null, null);
 536         }
 537
 538
 539         /**
 540          * Returns a {@link State} for a given uuid by first checking if the uuid has already been used in this import, if not
 541          * checking if the state exists in the database, if not creating it anew (with vocabulary etc.).
 542          * If label, text and labelAbbrev are all <code>null</code> no state is created.
 543          * @param importState
 544          * @param uuid
 545          * @param label
 546          * @param text
 547          * @param labelAbbrev
 548          * @param voc
 549          * @return
 550          */
 551         protected State getStateTerm(STATE importState, UUID uuid, String label, String text, String labelAbbrev, OrderedTermVocabulary<State> voc) {
 552                 if (uuid == null){
 553                         return null;
 554                 }
 555                 State stateTerm = importState.getStateTerm(uuid);
 556                 if (stateTerm == null){
 557                         stateTerm = CdmBase.deproxy(getTermService().find(uuid), State.class);
 558                         if (stateTerm == null && ! hasNoLabel(label, text, labelAbbrev)){
 559                                 stateTerm = State.NewInstance(text, label, labelAbbrev);
 560                                 stateTerm.setUuid(uuid);
 561                                 if (voc == null){
 562                                         boolean isOrdered = true;
 563                                         TermVocabulary<State> orderedVoc = getVocabulary(TermType.State, uuidUserDefinedStateVocabulary, "User defined vocabulary for states used by Categorical Data", "User Defined States", null, null, isOrdered, stateTerm);
 564                                         voc = CdmBase.deproxy(orderedVoc, OrderedTermVocabulary.class);
 565                                 }
 566                                 voc.addTerm(stateTerm);
 567                                 getTermService().save(stateTerm);
 568                         }else{
 569                                 logger.warn("No label provided for new state with uuid " + uuid);
 570                         }
 571                         importState.putStateTerm(stateTerm);
 572                 }
 573                 return stateTerm;
 574         }
 575
 576         /**
 577          * Returns a feature if it exists, null otherwise.
 578          * @see #getFeature(ImportStateBase, UUID, String, String, String, TermVocabulary)
 579          * @param state
 580          * @param uuid
 581          * @return
 582          */
 583         protected Feature getFeature(STATE state, UUID uuid){
 584                 return getFeature(state, uuid, null, null, null, null);
 585         }
 586
 587         /**
 588          * Returns a feature for a given uuid by first checking if the uuid has already been used in this import, if not
 589          * checking if the feature exists in the database, if not creating it anew (with vocabulary etc.).
 590          * If label, text and labelAbbrev are all <code>null</code> no feature is created.
 591          * @param state
 592          * @param uuid
 593          * @param label
 594          * @param text
 595          * @param labelAbbrev
 596          * @return
 597          */
 598         protected Feature getFeature(STATE state, UUID uuid, String label, String description, String labelAbbrev, TermVocabulary<Feature> voc){
 599                 if (uuid == null){
 600                         return null;
 601                 }
 602                 Feature feature = state.getFeature(uuid);
 603                 if (feature == null){
 604                         feature = (Feature)getTermService().find(uuid);
 605                         if (feature == null && ! hasNoLabel(label, description, labelAbbrev)){
 606                                 feature = Feature.NewInstance(description, label, labelAbbrev);
 607                                 feature.setUuid(uuid);
 608                                 feature.setSupportsTextData(true);
 609 //                              UUID uuidFeatureVoc = UUID.fromString("b187d555-f06f-4d65-9e53-da7c93f8eaa8");
 610                                 if (voc == null){
 611                                         boolean isOrdered = false;
 612                                         voc = getVocabulary(TermType.Feature, uuidUserDefinedFeatureVocabulary, "User defined vocabulary for features", "User Defined Features", null, null, isOrdered, feature);
 613                                 }
 614                                 voc.addTerm(feature);
 615                                 getTermService().save(feature);
 616                         }
 617                         state.putFeature(feature);
 618                 }
 619                 return feature;
 620         }
 621
 622         protected DefinedTerm getKindOfUnit(STATE state, UUID uuid, String label, String description, String labelAbbrev, TermVocabulary<DefinedTerm> voc){
 623                 if (uuid == null){
 624                         return null;
 625                 }
 626                 DefinedTerm unit = state.getKindOfUnit(uuid);
 627                 if (unit == null){
 628                         unit = (DefinedTerm)getTermService().find(uuid);
 629                         if (unit == null && ! hasNoLabel(label, description, labelAbbrev)){
 630                                 unit = DefinedTerm.NewKindOfUnitInstance(description, label, labelAbbrev);
 631                                 unit.setUuid(uuid);
 632                                 if (voc == null){
 633                                         boolean isOrdered = false;
 634                                         voc = getVocabulary(TermType.KindOfUnit, uuidUserDefinedKindOfUnitVocabulary, "User defined vocabulary for kind-of-units", "User Defined Measurement kind-of-units", null, null, isOrdered, unit);
 635                                 }
 636                                 voc.addTerm(unit);
 637                                 getTermService().save(unit);
 638                         }
 639                         state.putKindOfUnit(unit);
 640                 }
 641                 return unit;
 642         }
 643
 644         /**
 645          * Returns a {@link MeasurementUnit} for a given uuid by first checking if the uuid has already been used in this import, if not
 646          * checking if the {@link MeasurementUnit} exists in the database, if not creating it anew (with vocabulary etc.).
 647          * If label, text and labelAbbrev are all <code>null</code> no {@link MeasurementUnit} is created.
 648          * @param state
 649          * @param uuid
 650          * @param label
 651          * @param text
 652          * @param labelAbbrev
 653          * @return
 654          */
 655         protected MeasurementUnit getMeasurementUnit(STATE state, UUID uuid, String label, String description, String labelAbbrev, TermVocabulary<MeasurementUnit> voc){
 656                 if (uuid == null){
 657                         return null;
 658                 }
 659                 MeasurementUnit unit = state.getMeasurementUnit(uuid);
 660                 if (unit == null){
 661                         unit = (MeasurementUnit)getTermService().find(uuid);
 662                         if (unit == null && ! hasNoLabel(label, description, labelAbbrev)){
 663                                 unit = MeasurementUnit.NewInstance(description, label, labelAbbrev);
 664                                 unit.setUuid(uuid);
 665                                 if (voc == null){
 666                                         boolean isOrdered = false;
 667                                         voc = getVocabulary(TermType.MeasurementUnit, uuidUserDefinedMeasurementUnitVocabulary, "User defined vocabulary for measurement units", "User Defined Measurement Units", null, null, isOrdered, unit);
 668                                 }
 669                                 voc.addTerm(unit);
 670                                 getTermService().save(unit);
 671                         }
 672                         state.putMeasurementUnit(unit);
 673                 }
 674                 return unit;
 675         }
 676
 677         /**
 678          * Returns a {@link StatisticalMeasure} for a given uuid by first checking if the uuid has already been used in this import, if not
 679          * checking if the {@link StatisticalMeasure} exists in the database, if not creating it anew (with vocabulary etc.).
 680          * If label, text and labelAbbrev are all <code>null</code> no {@link StatisticalMeasure} is created.
 681          * @param state
 682          * @param uuid
 683          * @param label
 684          * @param text
 685          * @param labelAbbrev
 686          * @return
 687          */
 688         protected StatisticalMeasure getStatisticalMeasure(STATE state, UUID uuid, String label, String description, String labelAbbrev, TermVocabulary<StatisticalMeasure> voc){
 689                 if (uuid == null){
 690                         return null;
 691                 }
 692                 StatisticalMeasure statisticalMeasure = state.getStatisticalMeasure(uuid);
 693                 if (statisticalMeasure == null){
 694                         statisticalMeasure = (StatisticalMeasure)getTermService().find(uuid);
 695                         if (statisticalMeasure == null && ! hasNoLabel(label, description, labelAbbrev)){
 696                                 statisticalMeasure = StatisticalMeasure.NewInstance(description, label, labelAbbrev);
 697                                 statisticalMeasure.setUuid(uuid);
 698                                 if (voc == null){
 699                                         boolean isOrdered = false;
 700                                         voc = getVocabulary(TermType.StatisticalMeasure, uuidUserDefinedStatisticalMeasureVocabulary, "User defined vocabulary for statistical measures", "User Defined Statistical Measures", null, null, isOrdered, statisticalMeasure);
 701                                 }
 702                                 voc.addTerm(statisticalMeasure);
 703                                 getTermService().save(statisticalMeasure);
 704                         }
 705                         state.putStatisticalMeasure(statisticalMeasure);
 706                 }
 707                 return statisticalMeasure;
 708         }
 709
 710         /**
 711          * Returns a {@link Modifier} for a given uuid by first checking if the uuid has already been used in this import, if not
 712          * checking if the {@link Modifier} exists in the database, if not creating it anew (with vocabulary etc.).
 713          * If label, text and labelAbbrev are all <code>null</code> no {@link Modifier} is created.
 714          * @param state
 715          * @param uuid
 716          * @param label
 717          * @param text
 718          * @param labelAbbrev
 719          * @return
 720          */
 721         protected DefinedTerm getModifier(STATE state, UUID uuid, String label, String description, String labelAbbrev, TermVocabulary<DefinedTerm> voc){
 722                 if (uuid == null){
 723                         return null;
 724                 }
 725                 DefinedTerm modifier = state.getModifier(uuid);
 726                 if (modifier == null){
 727                         modifier = (DefinedTerm)getTermService().find(uuid);
 728                         if (modifier == null && ! hasNoLabel(label, description, labelAbbrev)){
 729                                 modifier = DefinedTerm.NewModifierInstance(description, label, labelAbbrev);
 730                                 modifier.setUuid(uuid);
 731                                 if (voc == null){
 732                                         boolean isOrdered = false;
 733                                         voc = getVocabulary(TermType.Modifier, uuidUserDefinedModifierVocabulary, "User defined vocabulary for modifier", "User Defined Modifier", null, null, isOrdered, modifier);
 734                                 }
 735                                 voc.addTerm(modifier);
 736                                 getTermService().save(modifier);
 737                         }
 738                         state.putModifier(modifier);
 739                 }
 740                 return modifier;
 741         }
 742
 743         /**
 744          * Returns a taxon relationship type for a given uuid by first checking if the uuid has already been used in this import, if not
 745          * checking if the taxon relationship type exists in the database, if not creating it anew (with vocabulary etc.).
 746          * If label, text and labelAbbrev are all <code>null</code> no taxon relationship type is created.
 747          * @param state
 748          * @param uuid
 749          * @param label
 750          * @param text
 751          * @param labelAbbrev
 752          * @return
 753          */
 754         protected TaxonRelationshipType getTaxonRelationshipType(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<TaxonRelationshipType> voc){
 755                 if (uuid == null){
 756                         return null;
 757                 }
 758                 TaxonRelationshipType relType = state.getTaxonRelationshipType(uuid);
 759                 if (relType == null){
 760                         relType = (TaxonRelationshipType)getTermService().find(uuid);
 761                         if (relType == null && ! hasNoLabel(label, text, labelAbbrev)){
 762                                 relType = TaxonRelationshipType.NewInstance(text, label, labelAbbrev, false, false);
 763                                 relType.setUuid(uuid);
 764                                 if (voc == null){
 765                                         boolean isOrdered = true;
 766                                         voc = getVocabulary(TermType.TaxonRelationshipType, uuidUserDefinedTaxonRelationshipTypeVocabulary, "User defined vocabulary for taxon relationship types", "User Defined Taxon Relationship Types", null, null, isOrdered, relType);
 767                                 }
 768                                 voc.addTerm(relType);
 769                                 getTermService().save(relType);
 770                         }
 771                         state.putTaxonRelationshipType(relType);
 772                 }
 773                 return relType;
 774         }
 775
 776         private boolean hasNoLabel(String label, String text, String labelAbbrev) {
 777                 return label == null && text == null && labelAbbrev == null;
 778         }
 779
 780
 781         /**
 782          * Returns a presence term for a given uuid by first ...
 783          * @param state
 784          * @param uuid
 785          * @param label
 786          * @param text
 787          * @param labelAbbrev
 788          * @return
 789          */
 790         protected PresenceTerm getPresenceTerm(STATE state, UUID uuid, String label, String text, String labelAbbrev){
 791                 if (uuid == null){
 792                         return null;
 793                 }
 794                 PresenceTerm presenceTerm = state.getPresenceTerm(uuid);
 795                 if (presenceTerm == null){
 796                         presenceTerm = (PresenceTerm)getTermService().find(uuid);
 797                         if (presenceTerm == null){
 798                                 presenceTerm = PresenceTerm.NewInstance(text, label, labelAbbrev);
 799                                 presenceTerm.setUuid(uuid);
 800                                 //set vocabulary ; FIXME use another user-defined vocabulary
 801                                 UUID uuidPresenceVoc = UUID.fromString("adbbbe15-c4d3-47b7-80a8-c7d104e53a05");
 802                                 TermVocabulary<PresenceTerm> voc = getVocabularyService().find(uuidPresenceVoc);
 803                                 voc.addTerm(presenceTerm);
 804                                 getTermService().save(presenceTerm);
 805                         }
 806                         state.putPresenceTerm(presenceTerm);
 807                 }
 808                 return presenceTerm;
 809         }
 810
 811         /**
 812          * Returns a language for a given uuid by first ...
 813          * @param state
 814          * @param uuid
 815          * @param label
 816          * @param text
 817          * @param labelAbbrev
 818          * @return
 819          */
 820         protected Language getLanguage(STATE state, UUID uuid, String label, String text, String labelAbbrev){
 821                 return getLanguage(state, uuid, label, text, labelAbbrev, null);
 822         }
 823
 824         protected Language getLanguage(STATE state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary voc){
 825                 if (uuid == null){
 826                         return null;
 827                 }
 828                 Language language = state.getLanguage(uuid);
 829                 if (language == null){
 830                         language = (Language)getTermService().find(uuid);
 831                         if (language == null){
 832                                 language = Language.NewInstance(text, label, labelAbbrev);
 833
 834                                 language.setUuid(uuid);
 835                                 if (voc == null){
 836                                         UUID uuidLanguageVoc = UUID.fromString("463a96f1-20ba-4a4c-9133-854c1682bd9b");
 837                                         boolean isOrdered = false;
 838                                         voc = getVocabulary(TermType.Language, uuidLanguageVoc, "User defined languages", "User defined languages", "User defined languages", null, isOrdered, language);
 839                                 }
 840                                 //set vocabulary ; FIXME use another user-defined vocabulary
 841
 842                                 voc.addTerm(language);
 843                                 getTermService().save(language);
 844                         }
 845                         state.putLanguage(language);
 846                 }
 847                 return language;
 848         }
 849
 850
 851         /**
 852          * @param uuid
 853          * @return
 854          *
 855          */
 856         protected <T extends DefinedTermBase> TermVocabulary<T> getVocabulary(TermType termType, UUID uuid, String description, String label, String abbrev, URI termSourceUri, boolean isOrdered, T type) {
 857                 List<String> propPath = Arrays.asList(new String[]{"terms"});
 858                 TermVocabulary<T> voc = getVocabularyService().load(uuid, propPath);
 859                 if (voc == null){
 860                         if (isOrdered){
 861                                 voc = OrderedTermVocabulary.NewInstance(termType, description, label, abbrev, termSourceUri);
 862                         }else{
 863                                 voc = TermVocabulary.NewInstance(termType, description, label, abbrev, termSourceUri);
 864                         }
 865                         voc.setUuid(uuid);
 866                         getVocabularyService().save(voc);
 867                 }
 868                 return voc;
 869         }
 870
 871         /**
 872          * Adds an orginal source to a sourceable objects (implemented for Identifiable entity and description element.
 873          * If cdmBase is not sourceable nothing happens.
 874          * TODO Move to DbImportBase once this exists.
 875          * TODO also implemented in DbImportObjectCreationMapper (reduce redundance)
 876          * @param rs
 877          * @param cdmBase
 878          * @param dbIdAttribute
 879          * @param namespace
 880          * @param citation
 881          * @throws SQLException
 882          */
 883         public void addOriginalSource(CdmBase cdmBase, Object idAttributeValue, String namespace, Reference citation)  {
 884                 if (cdmBase instanceof ISourceable ){
 885                         IOriginalSource source;
 886                         ISourceable sourceable = (ISourceable<?>)cdmBase;
 887                         Object id = idAttributeValue;
 888                         String strId = String.valueOf(id);
 889                         String microCitation = null;
 890                         OriginalSourceType type = OriginalSourceType.Import;
 891                         if (cdmBase instanceof IdentifiableEntity){
 892                                 source = IdentifiableSource.NewInstance(type, strId, namespace, citation, microCitation);
 893                         }else if (cdmBase instanceof DescriptionElementBase){
 894                                 source = DescriptionElementSource.NewInstance(type, strId, namespace, citation, microCitation);
 895                         }else{
 896                                 logger.warn("ISourceable not beeing identifiable entities or description element base are not yet supported. CdmBase is of type " + cdmBase.getClass().getName() + ". Original source not added.");
 897                                 return;
 898                         }
 899                         sourceable.addSource(source);
 900                 }else if (cdmBase != null){
 901                         logger.warn("Sourced object does not implement ISourceable: " + cdmBase.getClass() + "," + cdmBase.getUuid());
 902                 }else{
 903                         logger.warn("Sourced object is null");
 904                 }
 905         }
 906
 907         /**
 908          * @see #addOriginalSource(CdmBase, Object, String, Reference)
 909          * @param rs
 910          * @param cdmBase
 911          * @param dbIdAttribute
 912          * @param namespace
 913          * @param citation
 914          * @throws SQLException
 915          */
 916         public void addOriginalSource(ResultSet rs, CdmBase cdmBase, String dbIdAttribute, String namespace, Reference citation) throws SQLException {
 917                 Object id = rs.getObject(dbIdAttribute);
 918                 addOriginalSource(cdmBase, id, namespace, citation);
 919         }
 920
 921
 922         /**
 923          * If the child taxon is missing genus or species epithet information and the rank is below <i>genus</i>
 924          * or <i>species</i> respectively the according epithets are taken from the parent taxon.
 925          * If the name is an autonym and has no combination author/basionym author the authors are taken from
 926          * the parent.
 927          * @param parentTaxon
 928          * @param childTaxon
 929          */
 930         protected void fillMissingEpithetsForTaxa(Taxon parentTaxon, Taxon childTaxon) {
 931                 if (parentTaxon == null){
 932                         logger.warn("Parent taxon is null. Missing name parts can not be taken from parent");
 933                         return;
 934                 }
 935                 NonViralName<?> parentName = HibernateProxyHelper.deproxy(parentTaxon.getName(), NonViralName.class);
 936                 NonViralName<?> childName = HibernateProxyHelper.deproxy(childTaxon.getName(), NonViralName.class);
 937                 fillMissingEpithets(parentName, childName);
 938         }
 939
 940         /**
 941          * If the child name is missing genus or species epithet information and the rank is below <i>genus</i>
 942          * or <i>species</i> respectively the according epithets are taken from the parent name.
 943          * If the name is an autonym and has no combination author/basionym author the authors are taken from
 944          * the parent.
 945          * @param parentTaxon
 946          * @param childTaxon
 947          */
 948         protected void fillMissingEpithets(NonViralName parentName, NonViralName childName) {
 949                 if (StringUtils.isBlank(childName.getGenusOrUninomial()) && childName.getRank().isLower(Rank.GENUS()) ){
 950                         childName.setGenusOrUninomial(parentName.getGenusOrUninomial());
 951                 }
 952
 953                 if (StringUtils.isBlank(childName.getSpecificEpithet()) && childName.getRank().isLower(Rank.SPECIES()) ){
 954                         childName.setSpecificEpithet(parentName.getSpecificEpithet());
 955                 }
 956                 if (childName.isAutonym() && childName.getCombinationAuthorTeam() == null && childName.getBasionymAuthorTeam() == null ){
 957                         childName.setCombinationAuthorTeam(parentName.getCombinationAuthorTeam());
 958                         childName.setBasionymAuthorTeam(parentName.getBasionymAuthorTeam());
 959                 }
 960         }
 961
 962         /**
 963          * Returns the taxon description for a taxon. If there are multiple taxon descriptions
 964          * an arbitrary one is chosen.
 965          * If no taxon description exists, a new one is created if <code>createNewIfNotExists</code>
 966          * is <code>true</code>.
 967          * @param createNewIfNotExists
 968          * @param isImageGallery if true only taxon description being image galleries are considered.
 969          * If false only taxon description being no image galleries are considered.
 970          * @return
 971          */
 972         public TaxonNameDescription getTaxonNameDescription(TaxonNameBase name, boolean isImageGallery, boolean createNewIfNotExists) {
 973                 Reference<?> ref = null;
 974                 return getTaxonNameDescription(name, ref, isImageGallery, createNewIfNotExists);
 975         }
 976
 977         /**
 978          * Like {@link #getTaxonDescription(Taxon, boolean, boolean)}
 979          * Only matches a description if the given reference is a source of the description.<BR>
 980          * If a new description is created the given reference will be added as a source.
 981          *
 982          * @see #getTaxonDescription(Taxon, boolean, boolean)
 983          */
 984         public TaxonNameDescription getTaxonNameDescription(TaxonNameBase<?,?> name, Reference ref, boolean isImageGallery, boolean createNewIfNotExists) {
 985                 TaxonNameDescription result = null;
 986                 Set<TaxonNameDescription> descriptions= name.getDescriptions();
 987                 for (TaxonNameDescription description : descriptions){
 988                         if (description.isImageGallery() == isImageGallery){
 989                                 if (hasCorrespondingSource(ref, description)){
 990                                         result = description;
 991                                         break;
 992                                 }
 993                         }
 994                 }
 995                 if (result == null && createNewIfNotExists){
 996                         result = TaxonNameDescription.NewInstance(name);
 997                         result.setImageGallery(isImageGallery);
 998                         if (ref != null){
 999                                 result.addImportSource(null, null, ref, null);
1000                         }
1001                 }
1002                 return result;
1003         }
1004
1005         /**
1006          * Returns the taxon description for a taxon. If there are multiple taxon descriptions
1007          * an arbitrary one is chosen.
1008          * If no taxon description exists, a new one is created if <code>createNewIfNotExists</code>
1009          * is <code>true</code>.
1010          * @param createNewIfNotExists
1011          * @param isImageGallery if true only taxon description being image galleries are considered.
1012          * If false only taxon description being no image galleries are considered.
1013          * @return
1014          */
1015         public TaxonDescription getTaxonDescription(Taxon taxon, boolean isImageGallery, boolean createNewIfNotExists) {
1016                 Reference<?> ref = null;
1017                 return getTaxonDescription(taxon, ref, isImageGallery, createNewIfNotExists);
1018         }
1019
1020         /**
1021          * Like {@link #getTaxonDescription(Taxon, boolean, boolean)}
1022          * Only matches a description if the given reference is a source of the description.<BR>
1023          * If a new description is created the given reference will be added as a source.
1024          *
1025          * @see #getTaxonDescription(Taxon, boolean, boolean)
1026          */
1027         public TaxonDescription getTaxonDescription(Taxon taxon, Reference ref, boolean isImageGallery, boolean createNewIfNotExists) {
1028                 TaxonDescription result = null;
1029                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
1030                 for (TaxonDescription description : descriptions){
1031                         if (description.isImageGallery() == isImageGallery){
1032                                 if (hasCorrespondingSource(ref, description)){
1033                                         result = description;
1034                                         break;
1035                                 }
1036                         }
1037                 }
1038                 if (result == null && createNewIfNotExists){
1039                         result = TaxonDescription.NewInstance(taxon);
1040                         result.setImageGallery(isImageGallery);
1041                         if (ref != null){
1042                                 result.addImportSource(null, null, ref, null);
1043                         }
1044                 }
1045                 return result;
1046         }
1047
1048
1049         /**
1050          * Returns the {@link SpecimenDescription specimen description} for a {@link SpecimenOrObservationBase specimen or observation}.
1051          * If there are multiple specimen descriptions an arbitrary one is chosen.
1052          * If no specimen description exists, a new one is created if <code>createNewIfNotExists</code> is <code>true</code>.
1053          * @param createNewIfNotExists
1054          * @param isImageGallery if true only specimen description being image galleries are considered.
1055          * If false only specimen description being no image galleries are considered.
1056          * @return
1057          */
1058         public SpecimenDescription getSpecimenDescription(SpecimenOrObservationBase specimen, boolean isImageGallery, boolean createNewIfNotExists) {
1059                 Reference ref = null;
1060                 return getSpecimenDescription(specimen, ref, isImageGallery, createNewIfNotExists);
1061         }
1062
1063         /**
1064          * Like {@link #getSpecimenDescription(SpecimenOrObservationBase, boolean, boolean)}
1065          * Only matches a description if the given reference is a source of the description.<BR>
1066          * If a new description is created the given reference will be added as a source.
1067          *
1068          * @see #getTaxonDescription(Taxon, boolean, boolean)
1069          */
1070         public SpecimenDescription getSpecimenDescription(SpecimenOrObservationBase specimen, Reference ref, boolean isImageGallery, boolean createNewIfNotExists) {
1071                 SpecimenDescription result = null;
1072                 Set<SpecimenDescription> descriptions= specimen.getDescriptions();
1073                 for (SpecimenDescription description : descriptions){
1074                         if (description.isImageGallery() == isImageGallery){
1075                                 if (hasCorrespondingSource(ref, description)){
1076                                         result = description;
1077                                         break;
1078                                 }
1079                         }
1080                 }
1081                 if (result == null && createNewIfNotExists){
1082                         result = SpecimenDescription.NewInstance(specimen);
1083                         result.setImageGallery(isImageGallery);
1084                         if (ref != null){
1085                                 result.addImportSource(null, null, ref, null);
1086                         }
1087                 }
1088                 return result;
1089         }
1090
1091
1092         /**
1093          * Returns the textdata that holds general information about a feature for a taxon description.
1094          * This is mainly necessary for descriptions that have more than one description element for
1095          * a given feature such as 'distribution', 'description' or 'common name'. It may also hold
1096          * for hierarchical features where no description element exists for a higher hierarchy level.
1097          * Example: the description feature has subfeatures. But some information like authorship, figures,
1098          * sources need to be added to the description itself.
1099          * Currently a feature placeholder is marked by a marker of type 'feature placeholder'. Maybe in future
1100          * there will be a boolean marker in the TextData class itself.
1101          * @param state
1102          * @param feature
1103          * @param taxon
1104          * @param ref
1105          * @param createIfNotExists
1106          * @return
1107          */
1108         protected TextData getFeaturePlaceholder(STATE state, DescriptionBase<?> description, Feature feature, boolean createIfNotExists) {
1109                 UUID featurePlaceholderUuid = MarkupTransformer.uuidFeaturePlaceholder;
1110                 for (DescriptionElementBase element : description.getElements()){
1111                         if (element.isInstanceOf(TextData.class)){
1112                                 TextData textData = CdmBase.deproxy(element, TextData.class);
1113                                 if (textData.getFeature() == null || ! textData.getFeature().equals(feature)){
1114                                         continue;
1115                                 }
1116                                 for (Marker marker : textData.getMarkers()){
1117                                         MarkerType markerType = marker.getMarkerType();
1118                                         if (markerType != null &&
1119                                                         markerType.getUuid().equals(featurePlaceholderUuid) &&
1120                                                         marker.getValue() == true){
1121                                                 return textData;
1122                                         }
1123                                 }
1124                         }
1125                 }
1126                 if (createIfNotExists){
1127                         TextData newPlaceholder = TextData.NewInstance(feature);
1128                         MarkerType placeholderMarkerType = getMarkerType(state, featurePlaceholderUuid, "Feature Placeholder", "Feature Placeholder", null);
1129                         Marker marker = Marker.NewInstance(placeholderMarkerType, true);
1130                         newPlaceholder.addMarker(marker);
1131                         description.addElement(newPlaceholder);
1132                         return newPlaceholder;
1133                 }else{
1134                         return null;
1135                 }
1136         }
1137
1138
1139
1140         /**
1141          * Returns true, if this description has a source with a citation equal to the given reference.
1142          * Returns true if the given reference is null.
1143          * @param ref
1144          * @param description
1145          */
1146         private boolean hasCorrespondingSource(Reference<?> ref, DescriptionBase<?> description) {
1147                 if (ref != null){
1148                         for (IdentifiableSource source : description.getSources()){
1149                                 if (ref.equals(source.getCitation())){
1150                                         return true;
1151                                 }
1152                         }
1153                         return false;
1154                 }
1155                 return true;
1156
1157         }
1158
1159
1160         /**
1161          * Returns the accepted taxon of a {@link TaxonBase taxon base}. <BR>
1162          * If taxonBase is of type taxon the same object is returned. If taxonBase is of type
1163          * synonym the accepted taxon is returned if one exists. If no accepted taxon exists
1164          * <code>null</code> is returned. If multiple accepted taxa exist the one taxon with the
1165          * same secundum reference is returned. If no such single taxon exists an
1166          * {@link IllegalStateException illegal state exception} is thrown.
1167          * @param taxonBase
1168          * @return
1169          */
1170         protected Taxon getAcceptedTaxon(TaxonBase<?> taxonBase) {
1171                 if (taxonBase == null){
1172                         return null;
1173                 }else if(taxonBase.isInstanceOf(Taxon.class)){
1174                         return CdmBase.deproxy(taxonBase, Taxon.class);
1175                 }else if(taxonBase.isInstanceOf(Synonym.class)){
1176                         Synonym synonym = CdmBase.deproxy(taxonBase, Synonym.class);
1177                         Set<Taxon> acceptedTaxa = synonym.getAcceptedTaxa();
1178                         if (acceptedTaxa.size() == 0){
1179                                 return null;
1180                         }else if (acceptedTaxa.size() == 1){
1181                                 return acceptedTaxa.iterator().next();
1182                         }else{
1183                                 Reference<?> sec = synonym.getSec();
1184                                 if (sec != null){
1185                                         Set<Taxon> taxaWithSameSec = new HashSet<Taxon>();
1186                                         for (Taxon taxon: acceptedTaxa){
1187                                                 if (sec.equals(taxon.getSec())){
1188                                                         taxaWithSameSec.add(taxon);
1189                                                 }
1190                                         }
1191                                         if (taxaWithSameSec.size() == 1){
1192                                                 return taxaWithSameSec.iterator().next();
1193                                         }
1194                                 }
1195                                 throw new IllegalStateException("Can't define the one accepted taxon for a synonym out of multiple accept taxa");
1196                         }
1197                 }else{
1198                         throw new IllegalStateException("Unknown TaxonBase subclass: " + taxonBase.getClass().getName());
1199                 }
1200         }
1201
1202
1203
1204         /**
1205          * Creates
1206          * @param uriString
1207          * @param readDataFromUrl
1208          * @see #READ_MEDIA_DATA
1209          * @return
1210          * @throws MalformedURLException
1211          */
1212         protected Media getImageMedia(String uriString, boolean readMediaData) throws MalformedURLException {
1213                 if( uriString == null){
1214                         return null;
1215                 } else {
1216                         ImageInfo imageInfo = null;
1217                         URI uri;
1218                         uriString = uriString.replace(" ", "%20");  //replace whitespace
1219                         try {
1220                                 uri = new URI(uriString);
1221                                 try {
1222                                         if (readMediaData){
1223                                                 logger.warn(uri);
1224                                                 imageInfo = ImageInfo.NewInstance(uri, 0);
1225                                         }
1226                                 } catch (Exception e) {
1227                                         String message = "An error occurred when trying to read image meta data for " + uri.toString() + ": " +  e.getMessage();
1228                                         logger.warn(message);
1229                                         fireWarningEvent(message, "unknown location", 2, 0);
1230                                 }
1231                                 ImageFile imageFile = ImageFile.NewInstance(uri, null, imageInfo);
1232                                 MediaRepresentation representation = MediaRepresentation.NewInstance();
1233                                 if(imageInfo != null){
1234                                         representation.setMimeType(imageInfo.getMimeType());
1235                                         representation.setSuffix(imageInfo.getSuffix());
1236                                 }
1237                                 representation.addRepresentationPart(imageFile);
1238                                 Media media = Media.NewInstance();
1239                                 media.addRepresentation(representation);
1240                                 return media;
1241                         } catch (URISyntaxException e1) {
1242                                 String message = "An URISyntaxException occurred when trying to create uri from multimedia objcet string: " +  uriString;
1243                                 logger.warn(message);
1244                                 fireWarningEvent(message, "unknown location", 4, 0);
1245                                 return null;
1246                         }
1247                 }
1248         }
1249
1250
1251         /**
1252          * Retrieves an Integer value from a result set. If the value is NULL null is returned.
1253          * ResultSet.getInt() returns 0 therefore we need a special handling for this case.
1254          * @param rs
1255          * @param columnName
1256          * @return
1257          * @throws SQLException
1258          */
1259         protected Integer nullSafeInt(ResultSet rs, String columnName) throws SQLException {
1260                 Object intObject = rs.getObject(columnName);
1261                 if (intObject == null){
1262                         return null;
1263                 }else{
1264                         return Integer.valueOf(intObject.toString());
1265                 }
1266         }
1267
1268         protected Boolean nullSafeBoolean(ResultSet rs, String columnName) throws SQLException {
1269                 Object bitObject = rs.getObject(columnName);
1270                 if (bitObject == null){
1271                         return null;
1272                 }else{
1273                         return Boolean.valueOf(bitObject.toString());
1274                 }
1275         }
1276
1277         protected Double nullSafeDouble(ResultSet rs, String columnName) throws SQLException {
1278                 Object doubleObject = rs.getObject(columnName);
1279                 if (doubleObject == null){
1280                         return null;
1281                 }else{
1282                         return Double.valueOf(doubleObject.toString());
1283                 }
1284         }
1285
1286         protected Float nullSafeFloat(ResultSet rs, String columnName) throws SQLException {
1287                 Object doubleObject = rs.getObject(columnName);
1288                 if (doubleObject == null){
1289                         return null;
1290                 }else{
1291                         return Float.valueOf(doubleObject.toString());
1292                 }
1293         }
1294
1295
1296         /**
1297          * Returns <code>null</code> for all blank strings. Identity function otherwise.
1298          * @param str
1299          * @return
1300          */
1301         protected String NB(String str) {
1302                 if (StringUtils.isBlank(str)){
1303                         return null;
1304                 }else{
1305                         return str;
1306                 }
1307         }
1308
1309
1310 }