cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/sdd/in/SDDImport.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.sdd.in;
  11
  12 import java.io.File;
  13 import java.io.IOException;
  14 import java.net.MalformedURLException;
  15 import java.net.URI;
  16 import java.net.URL;
  17 import java.text.SimpleDateFormat;
  18 import java.util.ArrayList;
  19 import java.util.Date;
  20 import java.util.HashMap;
  21 import java.util.HashSet;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Set;
  25 import java.util.UUID;
  26
  27 import org.apache.commons.lang.StringUtils;
  28 import org.apache.log4j.Logger;
  29 import org.jdom.Element;
  30 import org.jdom.Namespace;
  31 import org.joda.time.DateTime;
  32 import org.springframework.stereotype.Component;
  33 import org.springframework.transaction.TransactionStatus;
  34
  35 import eu.etaxonomy.cdm.api.service.IDescriptionService;
  36 import eu.etaxonomy.cdm.common.media.ImageInfo;
  37 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  38 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  39 import eu.etaxonomy.cdm.io.common.ICdmImport;
  40 import eu.etaxonomy.cdm.io.common.ImportHelper;
  41 import eu.etaxonomy.cdm.io.common.XmlImportBase;
  42 import eu.etaxonomy.cdm.io.sdd.SDDTransformer;
  43 import eu.etaxonomy.cdm.model.agent.Person;
  44 import eu.etaxonomy.cdm.model.agent.Team;
  45 import eu.etaxonomy.cdm.model.common.Annotation;
  46 import eu.etaxonomy.cdm.model.common.AnnotationType;
  47 import eu.etaxonomy.cdm.model.common.CdmBase;
  48 import eu.etaxonomy.cdm.model.common.DefinedTerm;
  49 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  50 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  51 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  52 import eu.etaxonomy.cdm.model.common.Language;
  53 import eu.etaxonomy.cdm.model.common.LanguageString;
  54 import eu.etaxonomy.cdm.model.common.Marker;
  55 import eu.etaxonomy.cdm.model.common.MarkerType;
  56 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  57 import eu.etaxonomy.cdm.model.common.Representation;
  58 import eu.etaxonomy.cdm.model.common.TermBase;
  59 import eu.etaxonomy.cdm.model.common.TermType;
  60 import eu.etaxonomy.cdm.model.common.TermVocabulary;
  61 import eu.etaxonomy.cdm.model.common.VersionableEntity;
  62 import eu.etaxonomy.cdm.model.description.CategoricalData;
  63 import eu.etaxonomy.cdm.model.description.Feature;
  64 import eu.etaxonomy.cdm.model.description.FeatureNode;
  65 import eu.etaxonomy.cdm.model.description.FeatureTree;
  66 import eu.etaxonomy.cdm.model.description.MeasurementUnit;
  67 import eu.etaxonomy.cdm.model.description.QuantitativeData;
  68 import eu.etaxonomy.cdm.model.description.State;
  69 import eu.etaxonomy.cdm.model.description.StateData;
  70 import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
  71 import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
  72 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  73 import eu.etaxonomy.cdm.model.description.TextData;
  74 import eu.etaxonomy.cdm.model.description.WorkingSet;
  75 import eu.etaxonomy.cdm.model.location.NamedArea;
  76 import eu.etaxonomy.cdm.model.media.IdentifiableMediaEntity;
  77 import eu.etaxonomy.cdm.model.media.ImageFile;
  78 import eu.etaxonomy.cdm.model.media.Media;
  79 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
  80 import eu.etaxonomy.cdm.model.media.MediaRepresentationPart;
  81 import eu.etaxonomy.cdm.model.media.Rights;
  82 import eu.etaxonomy.cdm.model.name.NonViralName;
  83 import eu.etaxonomy.cdm.model.name.Rank;
  84 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  85 import eu.etaxonomy.cdm.model.occurrence.Specimen;
  86 import eu.etaxonomy.cdm.model.reference.Reference;
  87 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  88 import eu.etaxonomy.cdm.model.taxon.Classification;
  89 import eu.etaxonomy.cdm.model.taxon.Synonym;
  90 import eu.etaxonomy.cdm.model.taxon.Taxon;
  91 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  92
  93 /**
  94  * @author h.fradin
  95  * @created 24.10.2008
  96  * @version 1.0
  97  */
  98 @Component("sddImport")
  99 public class SDDImport extends XmlImportBase<SDDImportConfigurator, SDDImportState> implements ICdmImport<SDDImportConfigurator, SDDImportState> {
 100         private static final Logger logger = Logger.getLogger(SDDImport.class);
 101
 102         private static int modCount = 1000;
 103
 104         private UUID uuidAnnotationTypeLocation = UUID.fromString("a3737e07-72e3-46d2-986d-fa4cf5de0b63");
 105
 106
 107         private Map<String,Person> authors = new HashMap<String,Person>();
 108         private Map<String,String> citations = new HashMap<String,String>();
 109         private Map<String,String> defaultUnitPrefixes = new HashMap<String,String>();
 110         private Map<String,Person> editors = new HashMap<String,Person>();
 111         private Map<String,FeatureNode> featureNodes = new HashMap<String,FeatureNode>();
 112         private Map<String,Feature> features = new HashMap<String,Feature>();
 113         private Map<String,String> locations = new HashMap<String,String>();
 114         private Map<String,List<CdmBase>> mediaObject_ListCdmBase = new HashMap<String,List<CdmBase>>();
 115         private Map<String,String> mediaObject_Role = new HashMap<String,String>();
 116         private Map<String,Reference<?>> publications = new HashMap<String,Reference<?>>();
 117         private Map<String,State> states = new HashMap<String,State>();
 118         private Map<String,TaxonDescription> taxonDescriptions = new HashMap<String,TaxonDescription>();
 119         private Map<String,NonViralName<?>> taxonNameBases = new HashMap<String,NonViralName<?>>();
 120         private Map<String,MeasurementUnit> units = new HashMap<String,MeasurementUnit>();
 121         private Map<String,TaxonNode> taxonNodes = new HashMap<String,TaxonNode>();
 122         private Map<String,NamedArea> namedAreas = new HashMap<String,NamedArea>();
 123         private Map<String,Specimen> specimens = new HashMap<String,Specimen>();
 124         private Map<String,DefinedTerm> modifiers = new HashMap<String,DefinedTerm>();
 125
 126         private Set<MarkerType> markerTypes = new HashSet<MarkerType>();
 127         private Set<TermVocabulary<?>> vocabularies = new HashSet<TermVocabulary<?>>();
 128
 129         private Set<Feature> descriptiveConcepts = new HashSet<Feature>();
 130         private Set<AnnotationType> annotationTypes = new HashSet<AnnotationType>();
 131 //      private Set<Feature> featureSet = new HashSet<Feature>();
 132         private Set<Reference<?>> sources = new HashSet<Reference<?>>();
 133         private Reference<?> sec = ReferenceFactory.newDatabase();
 134         private Reference<?> sourceReference = null;
 135
 136         private Language datasetLanguage = null;
 137         private WorkingSet workingSet = null;
 138
 139         private Namespace xmlNamespace = Namespace.getNamespace("xml","http://www.w3.org/XML/1998/namespace");
 140
 141         private String generatorName = "";
 142         private String generatorVersion = "";
 143
 144
 145         private Set<StatisticalMeasure> statisticalMeasures = new HashSet<StatisticalMeasure>();
 146         private Set<VersionableEntity> featureData = new HashSet<VersionableEntity>();
 147         private Set<FeatureTree> featureTrees = new HashSet<FeatureTree>();
 148         private Set<Classification> classifications = new HashSet<Classification>();
 149
 150         private Rights copyright = null;
 151
 152         private int taxonNamesCount = 0;
 153
 154         public SDDImport(){
 155                 super();
 156         }
 157
 158         @Override
 159         public boolean doCheck(SDDImportState state){
 160                 boolean result = true;
 161                 logger.warn("No check implemented for SDD");
 162                 return result;
 163         }
 164
 165         //      @Override
 166         //      public boolean doInvoke(IImportConfigurator config, Map<String, MapWrapper<? extends CdmBase>> stores){
 167         @Override
 168         public void doInvoke(SDDImportState state){
 169
 170                 TransactionStatus ts = startTransaction();
 171                 SDDImportConfigurator sddConfig = state.getConfig();
 172                 IProgressMonitor progressMonitor = sddConfig.getProgressMonitor();
 173
 174                 logger.info("start Datasets ...");
 175
 176                 // <Datasets xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://rs.tdwg.org/UBIF/2006/" xsi:schemaLocation="http://rs.tdwg.org/UBIF/2006/ ../SDD.xsd">
 177                 Element root = sddConfig.getSourceRoot();
 178                 Namespace sddNamespace = sddConfig.getSddNamespace();
 179
 180                 logger.info("start TechnicalMetadata ...");
 181                 // <TechnicalMetadata created="2006-04-20T10:00:00">
 182                 importTechnicalMetadata(root, sddNamespace, sddConfig);
 183                 List<Element> elDatasets = root.getChildren("Dataset",sddNamespace);
 184 //              int i = 0;
 185
 186                 //for each Dataset
 187                 logger.info("start Dataset ...");
 188                 progressMonitor.beginTask("Importing SDD data", elDatasets.size());
 189                 for (Element elDataset : elDatasets){
 190                         importDataset(elDataset, sddNamespace, state);
 191 //                      if ((++i % modCount) == 0){ logger.info("dataset(s) handled: " + i);}
 192 //                      logger.info(i + " dataset(s) handled");
 193                         progressMonitor.worked(1);
 194                 }
 195                 commitTransaction(ts);
 196                 progressMonitor.done();
 197                 logger.info("End of transaction");
 198                 return;
 199         }
 200
 201         /* (non-Javadoc)
 202          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 203          */
 204         protected boolean isIgnore(SDDImportState state){
 205                 return false;
 206         }
 207
 208
 209         // associates the reference of a media object in SDD with a CdmBase Object
 210         protected void associateImageWithCdmBase(String refMO, CdmBase cb){
 211                 if ((refMO != null) && (cb!=null)) {
 212                         if (! refMO.equals("")) {
 213                                 if (! mediaObject_ListCdmBase.containsKey(refMO)) {
 214                                         List<CdmBase> lcb = new ArrayList<CdmBase>();
 215                                         lcb.add(cb);
 216                                         mediaObject_ListCdmBase.put(refMO,lcb);
 217                                 } else {
 218                                         List<CdmBase> lcb = mediaObject_ListCdmBase.get(refMO);
 219                                         lcb.add(cb);
 220                                         mediaObject_ListCdmBase.put(refMO,lcb);
 221                                 }
 222                         }
 223                 }
 224         }
 225
 226         // imports information about the Dataset
 227         protected void importDatasetRepresentation(Element parent, Namespace sddNamespace){
 228                 logger.info("start Representation ...");
 229                 /* <Representation>
 230                         <Label>The Genus Viola</Label>
 231                         <Detail>This is an example for a very simple SDD file, representing a single description with categorical, quantitative, and text character. Compare also the "Fragment*" examples, which contain more complex examples in the form of document fragments. Intended for version="SDD 1.1".</Detail>
 232                </Representation>
 233                  */
 234
 235
 236
 237                 Element elRepresentation = parent.getChild("Representation",sddNamespace);
 238                 String label = (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace);
 239                 String detail = (String)ImportHelper.getXmlInputValue(elRepresentation, "Detail",sddNamespace);
 240
 241                 //new
 242                 Representation representation = Representation.NewInstance(detail, label, null, datasetLanguage);
 243                 workingSet.addRepresentation(representation);
 244
 245
 246                 //old
 247 //              sec.setTitleCache(label, true);
 248 //
 249 //              if (detail != null) {
 250 //                      Annotation annotation = Annotation.NewInstance(detail, datasetLanguage);
 251 //                      annotation.setAnnotationType(AnnotationType.EDITORIAL());
 252 //                      sec.addAnnotation(annotation);
 253 //              }
 254
 255
 256                 List<Element> listMediaObjects = elRepresentation.getChildren("MediaObject",sddNamespace);
 257
 258                 for (Element elMediaObject : listMediaObjects) {
 259                         String ref = null;
 260                         String role = null;
 261                         if (elMediaObject != null) {
 262                                 ref = elMediaObject.getAttributeValue("ref");
 263                                 role = elMediaObject.getAttributeValue("role");
 264                         }
 265                         if (ref != null) {
 266                                 if (!ref.equals("")) {
 267                                         this.associateImageWithCdmBase(ref,sourceReference);
 268                                         this.associateImageWithCdmBase(ref,sec);
 269                                         mediaObject_Role.put(ref,role);
 270                                 }
 271                         }
 272                 }
 273         }
 274
 275         // imports the representation (label, detail, lang) of a particular SDD element
 276         protected void importRepresentation(Element parent, Namespace sddNamespace, VersionableEntity ve, String id, SDDImportState state){
 277                 Element elRepresentation = parent.getChild("Representation",sddNamespace);
 278
 279                 Map<Language,List<String>> langLabDet = new HashMap<Language,List<String>>();
 280
 281                 handleRepresentationLabels(sddNamespace, elRepresentation, langLabDet);
 282                 handleRepresentationDetails(sddNamespace, elRepresentation, langLabDet);
 283
 284                 if (ve instanceof TermBase) {
 285                         makeRepresentationForTerms((TermBase)ve, langLabDet);
 286                 }else if (ve instanceof Media) {
 287                         makeRepresentationForMedia((Media)ve, langLabDet);
 288                 }else if (ve instanceof IdentifiableEntity<?>) {
 289                         IdentifiableEntity<?> ie = (IdentifiableEntity<?>)ve;
 290                         makeRepresentationForIdentifiableEntity(sddNamespace, ie, elRepresentation, langLabDet);
 291                         if (ve instanceof IdentifiableMediaEntity<?>){
 292                                 makeRepresentationForIdentifiableMediaEntity(parent, sddNamespace, (IdentifiableMediaEntity<?>)ve);
 293                         }
 294                 }
 295
 296                 makeRepresentationMediaObjects(sddNamespace, ve, elRepresentation);//FIXME
 297
 298         }
 299
 300
 301         /**
 302          * Handles the "Detail" children of representations. Adds the result to the langLabDet.
 303          * @param sddNamespace
 304          * @param elRepresentation
 305          * @param langLabDet
 306          */
 307         private void handleRepresentationDetails(Namespace sddNamespace,
 308                         Element elRepresentation, Map<Language, List<String>> langLabDet) {
 309                 List<Element> listDetails = elRepresentation.getChildren("Detail",sddNamespace);
 310                 for (Element elDetail : listDetails){
 311                         Language language = getLanguage(elDetail);
 312                         String role = elDetail.getAttributeValue("role");
 313                         String detail = elDetail.getText();
 314                         List<String> labDet = langLabDet.get(language);
 315                         labDet.add(detail);
 316                         labDet.add(role);
 317                         langLabDet.put(language, labDet);
 318                 }
 319         }
 320
 321         /**
 322          * Handles the "Label" children of representations. Adds the result to the langLabDet.
 323          * @param sddNamespace
 324          * @param elRepresentation
 325          * @param langLabDet
 326          */
 327         private void handleRepresentationLabels(Namespace sddNamespace,
 328                                 Element elRepresentation, Map<Language, List<String>> langLabDet) {
 329                 // <Label xml:lang="la">Viola hederacea Labill.</Label>
 330                 List<Element> listLabels = elRepresentation.getChildren("Label",sddNamespace);
 331                 for (Element elLabel : listLabels){
 332                         Language language = getLanguage(elLabel);
 333                         String label = elLabel.getText();
 334                         List<String> labDet = new ArrayList<String>(3);
 335                         labDet.add(label);
 336                         langLabDet.put(language, labDet);
 337                 }
 338         }
 339
 340         /**
 341          *
 342          * @param media
 343          * @param langLabDet
 344          */
 345         private void makeRepresentationForMedia(Media media, Map<Language, List<String>> langLabDet) {
 346                 for (Language lang : langLabDet.keySet()){
 347                         List<String> labDet = langLabDet.get(lang);
 348                         if (labDet.get(0) != null){
 349                                 media.putTitle(LanguageString.NewInstance(labDet.get(0), lang));
 350                         }
 351                         if (labDet.size()>1) {
 352                                 media.putDescription(lang, labDet.get(1));
 353                         }
 354                 }
 355         }
 356
 357         /**
 358          * Handles representations for terms. Adds one representation per language in langLabDet.
 359          *
 360          * @param ve
 361          * @param langLabDet
 362          */
 363         private void makeRepresentationForTerms(TermBase tb, Map<Language, List<String>> langLabDet) {
 364                         for (Language lang : langLabDet.keySet()){
 365                                 List<String> labDet = langLabDet.get(lang);
 366                                 if (labDet.size()>0){
 367                                         if (labDet.size()>1) {
 368                                                 tb.addRepresentation(Representation.NewInstance(labDet.get(1), labDet.get(0), labDet.get(0), lang));
 369                                         } else {
 370                                                 tb.addRepresentation(Representation.NewInstance(labDet.get(0), labDet.get(0), labDet.get(0), lang));
 371                                         }
 372                                 }
 373                         }
 374         }
 375
 376
 377         /**
 378          * Handles the "MediaObject" children of representations.
 379          * @param sddNamespace
 380          * @param ve
 381          * @param elRepresentation
 382          */
 383         private void makeRepresentationMediaObjects(Namespace sddNamespace,     VersionableEntity ve, Element elRepresentation) {
 384                 List <Element> listMediaObjects = elRepresentation.getChildren("MediaObject", sddNamespace);
 385                 for (Element elMediaObject : listMediaObjects) {
 386                         String ref = null;
 387                         //TODO
 388                         String role = null;
 389                         if (elMediaObject != null) {
 390                                 ref = elMediaObject.getAttributeValue("ref");
 391                                 role = elMediaObject.getAttributeValue("role");
 392                         }
 393                         if (StringUtils.isNotBlank(ref)) {
 394                                 if (ve instanceof TaxonDescription) {
 395                                         TaxonDescription td = (TaxonDescription) ve;
 396                                         if (td.getSources().size() > 0) {
 397                                                 this.associateImageWithCdmBase(ref, td.getSources().iterator().next().getCitation());
 398                                         } else {
 399                                                 Reference<?> descriptionSource = ReferenceFactory.newGeneric();
 400                                                 sources.add(descriptionSource);
 401                                                 //TODO type
 402                                                 td.addSource(OriginalSourceType.Unknown, null, null, descriptionSource, null);
 403                                                 this.associateImageWithCdmBase(ref,descriptionSource);
 404                                         }
 405                                 } else {
 406                                         this.associateImageWithCdmBase(ref,ve);
 407                                 }
 408                         }
 409                 }
 410         }
 411
 412         /**
 413          * Handles the "Links" element
 414          * @param parent
 415          * @param sddNamespace
 416          * @param ve
 417          */
 418         private void makeRepresentationForIdentifiableMediaEntity(Element parent,
 419                         Namespace sddNamespace, IdentifiableMediaEntity ime) {
 420                 Element elLinks = parent.getChild("Links",sddNamespace);
 421
 422                 if (elLinks != null) {
 423
 424                         //  <Link rel="Alternate" href="http://www.diversitycampus.net/people/hagedorn"/>
 425                         List<Element> listLinks = elLinks.getChildren("Link", sddNamespace);
 426                         Media link = Media.NewInstance();
 427                         MediaRepresentation mr = MediaRepresentation.NewInstance();
 428                         int k = 0;
 429                         //for each Link
 430                         for (Element elLink : listLinks){
 431
 432                                 try {
 433                                         //TODO
 434                                         String rel = elLink.getAttributeValue("rel");
 435                                         String href = elLink.getAttributeValue("href");
 436                                         URI uri = new URI(href);
 437                                         mr.addRepresentationPart(MediaRepresentationPart.NewInstance(uri, null));
 438                                         link.addRepresentation(mr);
 439                                         ime.addMedia(link);
 440
 441                                 } catch (Exception e) {
 442                                         //FIXME
 443                                         logger.warn("Import of Link " + k + " failed.");
 444                                 }
 445
 446                                 if ((++k % modCount) == 0){ logger.info("Links handled: " + k);}
 447
 448                         }
 449                 }
 450         }
 451
 452         /**
 453          * @param sddNamespace
 454          * @param ve
 455          * @param elRepresentation
 456          * @param langLabDet
 457          * @return
 458          */
 459         private void makeRepresentationForIdentifiableEntity(Namespace sddNamespace, IdentifiableEntity<?> ie,
 460                                         Element elRepresentation, Map<Language, List<String>> langLabDet) {
 461                 List<String> labDet = null;
 462
 463                 if (ie instanceof TaxonNameBase) {
 464                         if (langLabDet.keySet().contains(getTermService().getLanguageByIso("la"))) {
 465                                 labDet = langLabDet.get(getTermService().getLanguageByIso("la"));
 466                         } else if (langLabDet.keySet().contains(datasetLanguage)) {
 467                                 labDet = langLabDet.get(datasetLanguage);
 468                                 logger.info("TaxonName " + (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace) + " is not specified as a latin name.");
 469                         } else {
 470                                 labDet = langLabDet.get(langLabDet.keySet().iterator().next());
 471                                 logger.info("TaxonName " + (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace) + " is not specified as a latin name.");
 472                         }
 473                 } else {
 474                         labDet = langLabDet.get(langLabDet.keySet().iterator().next());
 475                 }
 476
 477                 //FIXME labDet is != null only for TaxonNameBase
 478                 ie.setTitleCache(labDet.get(0), true);
 479
 480                 if (labDet.size()>1) {
 481                         Annotation annotation = null;
 482                         if (labDet.get(1) != null) {
 483                                 if (labDet.get(2) != null) {
 484                                         annotation = Annotation.NewInstance(labDet.get(2) + " - " + labDet.get(1), datasetLanguage);
 485                                 } else {
 486                                         annotation = Annotation.NewInstance(labDet.get(1), datasetLanguage);
 487                                 }
 488                         }
 489                         ie.addAnnotation(annotation);
 490                 }
 491                 return;
 492         }
 493
 494         /**
 495          * @param elLabel
 496          * @return
 497          */
 498         private Language getLanguage(Element elLanguage) {
 499                 String lang = elLanguage.getAttributeValue("lang",xmlNamespace);
 500                 Language language = null;
 501                 if (StringUtils.isNotBlank(lang)) {
 502                         language = getTermService().getLanguageByIso(lang.substring(0, 2));
 503                 } else {
 504                         language = datasetLanguage;
 505                 }
 506                 return language;
 507         }
 508
 509
 510         // imports the representation (label, detail, lang) of a particular SDD element
 511         protected void importTechnicalMetadata(Element root, Namespace sddNamespace, SDDImportConfigurator sddConfig){
 512                 Element elTechnicalMetadata = root.getChild("TechnicalMetadata", sddNamespace);
 513                 String nameCreated = elTechnicalMetadata.getAttributeValue("created");
 514                 sourceReference = sddConfig.getSourceReference();
 515
 516                 if (nameCreated != null) {
 517                         if (!nameCreated.equals("")) {
 518                                 int year = Integer.parseInt(nameCreated.substring(0,4));
 519                                 int monthOfYear = Integer.parseInt(nameCreated.substring(5,7));
 520                                 int dayOfMonth = Integer.parseInt(nameCreated.substring(8,10));
 521                                 int hourOfDay = Integer.parseInt(nameCreated.substring(11,13));
 522                                 int minuteOfHour = Integer.parseInt(nameCreated.substring(14,16));
 523                                 int secondOfMinute = Integer.parseInt(nameCreated.substring(17,19));
 524                                 DateTime created = new DateTime(year,monthOfYear,dayOfMonth,hourOfDay,minuteOfHour,secondOfMinute,0);
 525                                 sourceReference.setCreated(created);
 526                                 sec.setCreated(created);
 527                         }
 528                 }
 529
 530                 // <Generator name="n/a, handcrafted instance document" version="n/a"/>
 531                 Element elGenerator = elTechnicalMetadata.getChild("Generator", sddNamespace);
 532                 generatorName = elGenerator.getAttributeValue("name");
 533                 generatorVersion = elGenerator.getAttributeValue("version");
 534
 535                 sec.addAnnotation(Annotation.NewDefaultLanguageInstance(generatorName + " - " + generatorVersion));
 536                 sourceReference.addAnnotation(Annotation.NewDefaultLanguageInstance(generatorName + " - " + generatorVersion));
 537
 538         }
 539
 540         // imports the complete dataset information
 541         protected void importDataset(Element elDataset, Namespace sddNamespace, SDDImportState state){                  // <Dataset xml:lang="en-us">
 542
 543                 workingSet = WorkingSet.NewInstance();
 544                 importDatasetLanguage(elDataset,state);
 545                 importDatasetRepresentation(elDataset, sddNamespace);
 546                 importRevisionData(elDataset, sddNamespace);
 547                 importIPRStatements(elDataset, sddNamespace, state);
 548                 importTaxonNames(elDataset, sddNamespace, state);
 549
 550                 importDescriptiveConcepts(elDataset, sddNamespace, state);
 551                 importCharacters(elDataset, sddNamespace, state);
 552                 importCharacterTrees(elDataset, sddNamespace, state);
 553
 554                 MarkerType editorMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerEditor, "editor", "Editor", "edt");
 555                 MarkerType geographicAreaMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerSDDGeographicArea, "SDDGeographicArea", "SDDGeographicArea", "ga");
 556                 MarkerType descriptiveConceptMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerDescriptiveConcept, "DescriptiveConcept", "Descriptive Concept", "DC");
 557                 markerTypes.add(editorMarkerType);
 558                 markerTypes.add(geographicAreaMarkerType);
 559                 markerTypes.add(descriptiveConceptMarkerType);
 560
 561                 //saving of all imported data into the CDM db
 562                 saveVocabularies();
 563                 saveFeatures();
 564                 saveModifiers();
 565                 saveStates();
 566                 saveMarkerType();
 567                 saveAreas(geographicAreaMarkerType);
 568                 saveUnits();
 569                 saveStatisticalMeasure();
 570                 saveAnnotationType();
 571
 572                 importCodedDescriptions(elDataset, sddNamespace, state);
 573                 importAgents(elDataset, sddNamespace, state);
 574                 importPublications(elDataset, sddNamespace, state);
 575                 importMediaObjects(elDataset, sddNamespace, state);
 576                 importTaxonHierarchies(elDataset, sddNamespace, state);
 577                 importGeographicAreas(elDataset, sddNamespace, state);
 578                 importSpecimens(elDataset,sddNamespace, state);
 579
 580
 581                 if ((authors != null)||(editors != null)) {
 582                         Team team = Team.NewInstance();
 583                         if (authors != null) {
 584                                 for (Person author : authors.values()){
 585                                         team.addTeamMember(author);
 586                                 }
 587                         }
 588                         if (editors != null) {
 589                                 Marker marker = Marker.NewInstance();
 590                                 marker.setMarkerType(editorMarkerType);
 591                                 for (Person editor : editors.values()){
 592                                         Person edit = editor;
 593                                         edit.addMarker(marker);
 594                                         team.addTeamMember(edit);
 595                                 }
 596                         }
 597                         sec.setAuthorTeam(team);
 598                         sourceReference.setAuthorTeam(team);
 599                 }
 600
 601                 if (copyright != null) {
 602                         sourceReference.addRights(copyright);
 603                         sec.addRights(copyright);
 604                 }
 605
 606                 // Returns a CdmApplicationController created by the values of this configuration.
 607                 IDescriptionService descriptionService = getDescriptionService();
 608
 609                 for (TaxonDescription taxonDescription : taxonDescriptions.values()){
 610                         // Persists a Description
 611                         descriptionService.save(taxonDescription);
 612                 }
 613
 614                 for (String ref : taxonDescriptions.keySet()){
 615                         TaxonDescription td = taxonDescriptions.get(ref);
 616                         if (citations.containsKey(ref)) {
 617                                 Reference<?> publication = publications.get(citations.get(ref));
 618                                 if (locations.containsKey(ref)) {
 619                                         Annotation location = Annotation.NewInstance(locations.get(ref), datasetLanguage);
 620                                         //TODO move to a generic place (implemented in hurry therefore dirty)
 621                                         AnnotationType annotationType = getAnnotationType(state, uuidAnnotationTypeLocation, "location", "location", "location", null);
 622 //                                      annotationTypes.add(annotationType);  TODO necessary??
 623                                         location.setAnnotationType(annotationType);
 624                                         (publication).addAnnotation(location);
 625                                 }
 626                                 td.addSource(OriginalSourceType.PrimaryTaxonomicSource, null, null,publication, null);
 627                         }
 628                 }
 629                 logger.info("end makeTaxonDescriptions ...");
 630
 631                 if (descriptiveConcepts != null) {
 632                         for (Feature feature : descriptiveConcepts) {
 633                                 Marker marker = Marker.NewInstance();
 634                                 marker.setMarkerType(descriptiveConceptMarkerType);
 635                                 feature.addMarker(marker);
 636                         }
 637                 }
 638                 saveFeatures();
 639
 640                 for (Reference<?> publication : publications.values()){
 641                         getReferenceService().save(publication);
 642                 }
 643
 644                 for (Reference<?> source : sources){
 645                         getReferenceService().save(source);
 646                 }
 647
 648                 for (FeatureTree featureTree : featureTrees) {
 649                         getFeatureTreeService().save(featureTree);
 650                 }
 651                 getWorkingSetService().save(workingSet);
 652                 for (Classification classification : classifications) {
 653                         getClassificationService().save(classification);
 654                 }
 655                 for (Specimen specimen : specimens.values()) {
 656                         getOccurrenceService().save(specimen);
 657                 }
 658                 logger.info("end of persistence ...");
 659
 660                 return;
 661         }
 662
 663         /**
 664          *
 665          */
 666         private void saveVocabularies() {
 667                 for (TermVocabulary<?> vocabulary : vocabularies ){
 668                         getVocabularyService().save(vocabulary);
 669                 }
 670
 671         }
 672
 673         private void saveAnnotationType() {
 674                 for (AnnotationType annotationType: annotationTypes){
 675                         getTermService().saveOrUpdate(annotationType);
 676                 }
 677         }
 678
 679         private void saveStatisticalMeasure() {
 680                 for (StatisticalMeasure sm : statisticalMeasures){
 681                         getTermService().save(sm);
 682                 }
 683         }
 684
 685         private void saveUnits() {
 686                 if (units != null) {
 687                         for (MeasurementUnit unit : units.values()){
 688                                 if (unit != null) {
 689                                         getTermService().save(unit);
 690                                 }
 691                         }
 692                 }
 693         }
 694
 695         private void saveAreas(MarkerType geographicAreaMarkerType) {
 696                 for (NamedArea area : namedAreas.values() ){
 697                         Marker marker = Marker.NewInstance();
 698                         marker.setMarkerType(geographicAreaMarkerType);
 699                         area.addMarker(marker);
 700                         getTermService().save(area);
 701                 }
 702         }
 703
 704         private void saveStates() {
 705                 for (State state : states.values() ){
 706                         getTermService().save(state);
 707                 }
 708         }
 709
 710         private void saveMarkerType() {
 711                 for (MarkerType markerType : markerTypes){
 712                         getTermService().save(markerType);
 713                 }
 714         }
 715
 716         private void saveModifiers() {
 717                 for (DefinedTerm modifier : modifiers.values() ){
 718                         getTermService().save(modifier);
 719                 }
 720         }
 721
 722         private void saveFeatures() {
 723                 for (Feature feature : features.values() ){
 724                         getTermService().save(feature);
 725                 }
 726         }
 727
 728         // imports the default language of the dataset
 729         protected void importDatasetLanguage(Element elDataset, SDDImportState state){
 730                 String nameLang = elDataset.getAttributeValue("lang",xmlNamespace);
 731
 732                 if (StringUtils.isNotBlank(nameLang)) {
 733                         String iso = nameLang.substring(0, 2);
 734                         datasetLanguage = getTermService().getLanguageByIso(iso);
 735                 } else {
 736                         datasetLanguage = Language.DEFAULT();
 737                 }
 738                 if (datasetLanguage == null) {
 739                         datasetLanguage = Language.DEFAULT();
 740                 }
 741         }
 742
 743         // imports the specimens
 744         protected void importSpecimens(Element elDataset, Namespace sddNamespace, SDDImportState cdmState) {
 745                 logger.info("start Specimens ...");
 746                 /*      <Specimens>
 747                         <Specimen id="sp1">
 748                                 <Representation>
 749                                 <Label>TJM45337</Label>
 750                                 </Representation>
 751                         </Specimen>
 752                 </Specimens>
 753                  */
 754                 Element elSpecimens = elDataset.getChild("Specimens",sddNamespace);
 755                 if (elSpecimens != null){
 756                         List<Element> listSpecimens = elSpecimens.getChildren("Specimen", sddNamespace);
 757                         for (Element elSpecimen : listSpecimens) {
 758                                 String id = elSpecimen.getAttributeValue("id");
 759                                 Specimen specimen = null;
 760                                 if (!id.equals("")) {
 761                                         specimen = Specimen.NewInstance();
 762                                         specimens.put(id,specimen);
 763                                         importRepresentation(elSpecimen, sddNamespace, specimen, id, cdmState);
 764                                 }
 765                         }
 766
 767                 }
 768         }
 769
 770         // imports the revision data associated with the Dataset (authors, modifications)
 771         protected void importRevisionData(Element elDataset, Namespace sddNamespace){
 772                 // <RevisionData>
 773                 logger.info("start RevisionData ...");
 774                 Element elRevisionData = elDataset.getChild("RevisionData",sddNamespace);
 775                 if (elRevisionData != null){
 776                         // <Creators>
 777                         Element elCreators = elRevisionData.getChild("Creators",sddNamespace);
 778
 779                         // <Agent role="aut" ref="a1"/>
 780                         List<Element> listAgents = elCreators.getChildren("Agent", sddNamespace);
 781
 782                         int j = 0;
 783                         //for each Agent
 784                         for (Element elAgent : listAgents){
 785
 786                                 String role = elAgent.getAttributeValue("role");
 787                                 String ref = elAgent.getAttributeValue("ref");
 788                                 if (role.equals("aut")) {
 789                                         if(!ref.equals("")) {
 790                                                 authors.put(ref, null);
 791                                         }
 792                                 }
 793                                 if (role.equals("edt")) {
 794                                         if(!ref.equals("")) {
 795                                                 editors.put(ref, null);
 796                                         }
 797                                 }
 798                                 if ((++j % modCount) == 0){ logger.info("Agents handled: " + j);}
 799
 800                         }
 801
 802                         // <DateModified>2006-04-08T00:00:00</DateModified>
 803                         String stringDateModified = (String)ImportHelper.getXmlInputValue(elRevisionData, "DateModified",sddNamespace);
 804
 805                         if (stringDateModified != null) {
 806                                 SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss");
 807                                 Date d = null;
 808                                 try {
 809                                         d = sdf.parse(stringDateModified);
 810                                 } catch(Exception e) {
 811                                         System.err.println("Exception :");
 812                                         e.printStackTrace();
 813                                 }
 814
 815                                 DateTime updated = null;
 816                                 if (d != null) {
 817                                         updated = new DateTime(d);
 818                                         sourceReference.setUpdated(updated);
 819                                         sec.setUpdated(updated);
 820                                 }
 821                         }
 822                 }
 823         }
 824
 825         // imports ipr statements associated with a dataset
 826         protected void importIPRStatements(Element elDataset, Namespace sddNamespace, SDDImportState state){
 827                 // <IPRStatements>
 828                 logger.info("start IPRStatements ...");
 829                 Element elIPRStatements = elDataset.getChild("IPRStatements",sddNamespace);
 830                 // <IPRStatement role="Copyright">
 831                 if (elIPRStatements != null) {
 832                         List<Element> listIPRStatements = elIPRStatements.getChildren("IPRStatement", sddNamespace);
 833                         int j = 0;
 834                         //for each IPRStatement
 835
 836                         for (Element elIPRStatement : listIPRStatements){
 837
 838                                 String role = elIPRStatement.getAttributeValue("role");
 839                                 // <Label xml:lang="en-au">(c) 2003-2006 Centre for Occasional Botany.</Label>
 840                                 Element elLabel = elIPRStatement.getChild("Label",sddNamespace);
 841                                 String lang = "";
 842                                 if (elLabel != null) {
 843                                         lang = elLabel.getAttributeValue("lang",xmlNamespace);
 844                                 }
 845                                 String label = (String)ImportHelper.getXmlInputValue(elIPRStatement, "Label",sddNamespace);
 846
 847                                 if (role.equals("Copyright")) {
 848                                         Language iprLanguage = null;
 849                                         if (lang != null) {
 850                                                 if (!lang.equals("")) {
 851                                                         iprLanguage = getTermService().getLanguageByIso(lang.substring(0, 2));
 852                                                 } else {
 853                                                         iprLanguage = datasetLanguage;
 854                                                 }
 855                                         }
 856                                         if (iprLanguage == null) {
 857                                                 iprLanguage = datasetLanguage;
 858                                         }
 859                                         copyright = Rights.NewInstance(label, iprLanguage);
 860                                 }
 861
 862                                 if (copyright != null) {
 863                                         sourceReference.addRights(copyright);
 864                                         sec.addRights(copyright);
 865                                 }
 866
 867                                 if ((++j % modCount) == 0){ logger.info("IPRStatements handled: " + j);}
 868
 869                         }
 870                 }
 871         }
 872
 873         // imports the taxon names
 874         protected void importTaxonNames(Element elDataset, Namespace sddNamespace, SDDImportState state){
 875                 // <TaxonNames>
 876                 logger.info("start TaxonNames ...");
 877                 Element elTaxonNames = elDataset.getChild("TaxonNames",sddNamespace);
 878                 // <TaxonName id="t1" uri="urn:lsid:authority:namespace:my-own-id">
 879                 if (elTaxonNames != null) {
 880                         List<Element> listTaxonNames = elTaxonNames.getChildren("TaxonName", sddNamespace);
 881                         int j = 0;
 882                         //for each TaxonName
 883                         for (Element elTaxonName : listTaxonNames){
 884
 885                                 String id = elTaxonName.getAttributeValue("id");
 886                                 String uri = elTaxonName.getAttributeValue("uri");
 887
 888                                 NonViralName<?> tnb = null;
 889                                 if (!id.equals("")) {
 890                                         tnb = NonViralName.NewInstance(Rank.UNKNOWN_RANK());
 891                                         IdentifiableSource source = null;
 892                                         if (uri != null) {
 893                                                 if (! isNotBlank(uri)) {
 894                                                         //TODO type
 895                                                         source = IdentifiableSource.NewInstance(OriginalSourceType.Unknown, id, "TaxonName", ReferenceFactory.newGeneric(), uri);
 896                                                 }
 897                                         } else {
 898                                                 source = IdentifiableSource.NewDataImportInstance(id, "TaxonName");
 899                                         }
 900                                         tnb.addSource(source);
 901                                         taxonNameBases.put(id,tnb);
 902                                 }
 903
 904                                 // <Representation>
 905                                 // <Label xml:lang="la">Viola hederacea Labill.</Label>
 906                                 importRepresentation(elTaxonName, sddNamespace, tnb, id, state);
 907
 908                                 if ((++j % modCount) == 0){ logger.info("TaxonNames handled: " + j);}
 909
 910                         }
 911                 }
 912         }
 913
 914         // imports the characters (categorical, quantitative and text ; sequence characters not supported) which correspond to CDM Features
 915         protected void importCharacters(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
 916                 // <Characters>
 917                 logger.info("start Characters ...");
 918                 Element elCharacters = elDataset.getChild("Characters", sddNamespace);
 919
 920                 // <CategoricalCharacter id="c1">
 921                 if (elCharacters != null) {
 922                         handleCategoricalData(sddNamespace, cdmState, elCharacters);
 923                         handleQuantitativeData(sddNamespace, cdmState, elCharacters);
 924                         handleTextCharacters(sddNamespace, cdmState, elCharacters);
 925                 }
 926
 927                 /*for (Iterator<Feature> f = features.values().iterator() ; f.hasNext() ;){
 928                         featureSet.add(f.next()); //XIM Why this line ?
 929                 }*/
 930
 931                 return;
 932
 933         }
 934
 935         /**
 936          * @param sddNamespace
 937          * @param sddConfig
 938          * @param elCharacters
 939          * @return
 940          */
 941         private void handleCategoricalData(Namespace sddNamespace, SDDImportState cdmState, Element elCharacters) {
 942                 List<Element> elCategoricalCharacters = elCharacters.getChildren("CategoricalCharacter", sddNamespace);
 943                 int j = 0;
 944                 for (Element elCategoricalCharacter : elCategoricalCharacters){
 945                         try {
 946
 947                                 String idCC = elCategoricalCharacter.getAttributeValue("id");
 948                                 Feature categoricalCharacter = Feature.NewInstance();
 949                                 categoricalCharacter.setKindOf(Feature.DESCRIPTION());
 950                                 importRepresentation(elCategoricalCharacter, sddNamespace, categoricalCharacter, idCC, cdmState);
 951                                 categoricalCharacter.setSupportsCategoricalData(true);
 952
 953                                 // <States>
 954                                 Element elStates = elCategoricalCharacter.getChild("States",sddNamespace);
 955
 956                                 // <StateDefinition id="s1">
 957                                 List<Element> elStateDefinitions = elStates.getChildren("StateDefinition",sddNamespace);
 958                                 TermVocabulary<State> termVocabularyState = TermVocabulary.NewInstance(TermType.State, null, null, null, null);
 959
 960                                 vocabularies.add(termVocabularyState);
 961
 962                                 int k = 0;
 963                                 //for each StateDefinition
 964                                 for (Element elStateDefinition : elStateDefinitions){
 965
 966                                         if ((++k % modCount) == 0){ logger.info("StateDefinitions handled: " + (k-1));}
 967
 968                                         String idS = elStateDefinition.getAttributeValue("id");
 969                                         State state = states.get(idS);
 970                                         if (state == null){
 971                                                 state = State.NewInstance();
 972                                         }else{
 973                                                 logger.debug("State duplicate found");
 974                                         }
 975                                         importRepresentation(elStateDefinition, sddNamespace, state, idS, cdmState);
 976
 977                                         termVocabularyState.addTerm(state);
 978                                         states.put(idS,state);
 979                                 }
 980                                 categoricalCharacter.addSupportedCategoricalEnumeration(termVocabularyState);
 981                                 features.put(idCC, categoricalCharacter);
 982
 983                         } catch (Exception e) {
 984                                 logger.warn("Import of CategoricalCharacter " + j + " failed.");
 985                                 cdmState.setUnsuccessfull();
 986                         }
 987
 988                         if ((++j % modCount) == 0){ logger.info("CategoricalCharacters handled: " + j);}
 989
 990                 }
 991                 return;
 992         }
 993
 994         /**
 995          * @param sddNamespace
 996          * @param sddConfig
 997          * @param elCharacters
 998          */
 999         private void handleQuantitativeData(Namespace sddNamespace,     SDDImportState cdmState, Element elCharacters) {
1000                 int j;
1001                 // <QuantitativeCharacter id="c2">
1002                 List<Element> elQuantitativeCharacters = elCharacters.getChildren("QuantitativeCharacter", sddNamespace);
1003                 j = 0;
1004                 //for each QuantitativeCharacter
1005                 for (Element elQuantitativeCharacter : elQuantitativeCharacters){
1006
1007                         try {
1008
1009                                 String idQC = elQuantitativeCharacter.getAttributeValue("id");
1010
1011                                 // <Representation>
1012                                 //  <Label>Leaf length</Label>
1013                                 // </Representation>
1014                                 Feature quantitativeCharacter = Feature.NewInstance();
1015                                 quantitativeCharacter.setKindOf(Feature.DESCRIPTION());
1016                                 importRepresentation(elQuantitativeCharacter, sddNamespace, quantitativeCharacter, idQC, cdmState);
1017
1018                                 quantitativeCharacter.setSupportsQuantitativeData(true);
1019
1020                                 // <MeasurementUnit>
1021                                 //  <Label role="Abbrev">m</Label>
1022                                 // </MeasurementUnit>
1023                                 Element elMeasurementUnit = elQuantitativeCharacter.getChild("MeasurementUnit",sddNamespace);
1024                                 String label = "";
1025                                 String role = "";
1026                                 if (elMeasurementUnit != null) {
1027                                         Element elLabel = elMeasurementUnit.getChild("Label",sddNamespace);
1028                                         role = elLabel.getAttributeValue("role");
1029                                         label = (String)ImportHelper.getXmlInputValue(elMeasurementUnit, "Label",sddNamespace);
1030                                 }
1031
1032                                 MeasurementUnit unit = null;
1033                                 if (!label.equals("")){
1034                                         if (role != null) {
1035                                                 if (role.equals("Abbrev")){
1036                                                         unit = MeasurementUnit.NewInstance(label,label,label);
1037                                                 }
1038                                         } else {
1039                                                 unit = MeasurementUnit.NewInstance(label,label,label);
1040                                         }
1041                                 }
1042
1043                                 if (unit != null) {
1044                                         units.put(idQC, unit);
1045                                 }
1046
1047                                 //<Default>
1048                                 //  <MeasurementUnitPrefix>milli</MeasurementUnitPrefix>
1049                                 //</Default>
1050                                 Element elDefault = elQuantitativeCharacter.getChild("Default",sddNamespace);
1051                                 if (elDefault != null) {
1052                                         String measurementUnitPrefix = (String)ImportHelper.getXmlInputValue(elDefault, "MeasurementUnitPrefix",sddNamespace);
1053                                         if (! measurementUnitPrefix.equals("")){
1054                                                 defaultUnitPrefixes.put(idQC, measurementUnitPrefix);
1055                                         }
1056                                 }
1057
1058                                 features.put(idQC, quantitativeCharacter);
1059
1060                         } catch (Exception e) {
1061                                 //FIXME
1062                                 logger.warn("Import of QuantitativeCharacter " + j + " failed.");
1063                                 cdmState.setUnsuccessfull();
1064                         }
1065
1066                         if ((++j % modCount) == 0){ logger.info("QuantitativeCharacters handled: " + j);}
1067
1068                 }
1069                 return;
1070         }
1071
1072         private void handleTextCharacters(Namespace sddNamespace, SDDImportState cdmState, Element elCharacters) {
1073                 int j;
1074                 // <TextCharacter id="c3">
1075                 List<Element> elTextCharacters = elCharacters.getChildren("TextCharacter", sddNamespace);
1076                 j = 0;
1077                 //for each TextCharacter
1078                 for (Element elTextCharacter : elTextCharacters){
1079
1080                         try {
1081
1082                                 String idTC = elTextCharacter.getAttributeValue("id");
1083
1084                                 // <Representation>
1085                                 //  <Label xml:lang="en">Leaf features not covered by other characters</Label>
1086                                 // </Representation>
1087                                 Feature textCharacter = Feature.NewInstance();
1088                                 textCharacter.setKindOf(Feature.DESCRIPTION());
1089                                 importRepresentation(elTextCharacter, sddNamespace, textCharacter, idTC, cdmState);
1090
1091                                 textCharacter.setSupportsTextData(true);
1092
1093                                 features.put(idTC, textCharacter);
1094
1095                         } catch (Exception e) {
1096                                 //FIXME
1097                                 logger.warn("Import of TextCharacter " + j + " failed.");
1098                                 cdmState.setUnsuccessfull();
1099                         }
1100
1101                         if ((++j % modCount) == 0){ logger.info("TextCharacters handled: " + j);}
1102
1103                 }
1104                 return;
1105         }
1106
1107         // imports the descriptions of taxa
1108         protected void importCodedDescriptions(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1109
1110                 // <CodedDescriptions>
1111                 logger.info("start CodedDescriptions ...");
1112                 Element elCodedDescriptions = elDataset.getChild("CodedDescriptions",sddNamespace);
1113
1114                 // <CodedDescription id="D101">
1115                 if (elCodedDescriptions != null) {
1116                         List<Element> listCodedDescriptions = elCodedDescriptions.getChildren("CodedDescription", sddNamespace);
1117                         int j = 0;
1118                         //for each CodedDescription
1119                         for (Element elCodedDescription : listCodedDescriptions){
1120                                 handleCodedDescription(sddNamespace, cdmState, elCodedDescription, j);
1121                                 if ((++j % modCount) == 0){ logger.info("CodedDescriptions handled: " + j);}
1122                         }
1123                 }
1124                 return;
1125         }
1126
1127         /**
1128          * @param sddNamespace
1129          * @param sddConfig
1130          * @param j
1131          * @param elCodedDescription
1132          * @return
1133          */
1134         private void handleCodedDescription(Namespace sddNamespace, SDDImportState cdmState, Element elCodedDescription, int j) {
1135                 try {
1136
1137                         String idCD = elCodedDescription.getAttributeValue("id");
1138
1139                         // <Representation>
1140                         //  <Label>&lt;i&gt;Viola hederacea&lt;/i&gt; Labill. as revised by R. Morris April 8, 2006</Label>
1141                         // </Representation>
1142                         TaxonDescription taxonDescription = TaxonDescription.NewInstance();
1143                         if (!generatorName.isEmpty()){
1144                                 Annotation annotation = Annotation.NewInstance(generatorName, AnnotationType.TECHNICAL(),Language.DEFAULT());
1145                                 taxonDescription.addAnnotation(annotation);
1146                         }
1147                         importRepresentation(elCodedDescription, sddNamespace, taxonDescription, idCD, cdmState);
1148
1149                         // <Scope>
1150                         //  <TaxonName ref="t1"/>
1151                         //  <Citation ref="p1" location="p. 30"/>
1152                         // </Scope>
1153                         Element elScope = elCodedDescription.getChild("Scope", sddNamespace);
1154                         Taxon taxon;
1155                         if (elScope != null) {
1156                                 taxon = handleCDScope(sddNamespace, cdmState, idCD, elScope);
1157                         } else {//in case no taxon is linked to the description, a new one is created
1158                                 taxon = handleCDNoScope(sddNamespace, cdmState, elCodedDescription);
1159                         }
1160
1161                         // <SummaryData>
1162                         Element elSummaryData = elCodedDescription.getChild("SummaryData",sddNamespace);
1163                         if (elSummaryData != null) {
1164                                 handleSummaryCategoricalData(sddNamespace, taxonDescription, elSummaryData);
1165                                 handleSummaryQuantitativeData(sddNamespace, taxonDescription, elSummaryData);
1166                                 handleSummaryTextData(sddNamespace, taxonDescription, elSummaryData);
1167                         }
1168
1169                         if (taxon != null) {
1170                                 taxon.addDescription(taxonDescription);
1171                         }
1172 //
1173                         workingSet.addDescription(taxonDescription);
1174
1175 //OLD                   taxonDescription.setDescriptiveSystem(featureSet);
1176
1177                         taxonDescriptions.put(idCD, taxonDescription);//FIXME
1178
1179                 } catch (Exception e) {
1180                         //FIXME
1181                         logger.warn("Import of CodedDescription " + j + " failed.", e);
1182                         cdmState.setUnsuccessfull();
1183                 }
1184                 return;
1185         }
1186
1187         /**
1188          * @param sddNamespace
1189          * @param sddConfig
1190          * @param elCodedDescription
1191          * @param taxon
1192          * @return
1193          */
1194         private Taxon handleCDNoScope(Namespace sddNamespace, SDDImportState cdmState, Element elCodedDescription       ) {
1195                 Taxon taxon = null;
1196                 NonViralName<?> nonViralName = NonViralName.NewInstance(Rank.UNKNOWN_RANK());
1197                 String id = new String("" + taxonNamesCount);
1198                 IdentifiableSource source = IdentifiableSource.NewDataImportInstance(id, "TaxonName");
1199                 importRepresentation(elCodedDescription, sddNamespace, nonViralName, id, cdmState);
1200
1201                 if(cdmState.getConfig().isDoMatchTaxa()){
1202                         taxon = getTaxonService().findBestMatchingTaxon(nonViralName.getTitleCache());
1203                 }
1204
1205                 if(taxon != null){
1206                         nonViralName = HibernateProxyHelper.deproxy(taxon.getName(), NonViralName.class);
1207 //                                                      taxonNameBases.put(id ,tnb);
1208 //                                                      taxonNamesCount++;
1209                         logger.info("using existing Taxon " + taxon.getTitleCache());
1210                 } else {
1211                         nonViralName.addSource(source);
1212                         taxonNameBases.put(id ,nonViralName);
1213                         taxonNamesCount++;
1214                         logger.info("creating new Taxon from TaxonName " + nonViralName.getTitleCache());
1215                         taxon = Taxon.NewInstance(nonViralName, sec);
1216                 }
1217                 return taxon;
1218         }
1219
1220         /**
1221          * @param sddNamespace
1222          * @param sddConfig
1223          * @param idCD
1224          * @param elScope
1225          * @param taxon
1226          * @return
1227          */
1228         private Taxon handleCDScope(Namespace sddNamespace, SDDImportState cdmState,
1229                         String idCD, Element elScope) {
1230                 Taxon taxon = null;
1231                 Element elTaxonName = elScope.getChild("TaxonName", sddNamespace);
1232                 String ref = elTaxonName.getAttributeValue("ref");
1233                 NonViralName<?> nonViralName = taxonNameBases.get(ref);
1234
1235                 if(cdmState.getConfig().isDoMatchTaxa()){
1236                         taxon = getTaxonService().findBestMatchingTaxon(nonViralName.getTitleCache());
1237                 }
1238
1239                 if(taxon != null){
1240                         logger.info("using existing Taxon" + taxon.getTitleCache());
1241                         if(!nonViralName.getUuid().equals(taxon.getName().getUuid())){
1242                                 logger.warn("TaxonNameBase entity of existing taxon does not match Name in list -> replacing Name in list");
1243                                 nonViralName = HibernateProxyHelper.deproxy(taxon.getName(), NonViralName.class);
1244                         }
1245                 } else {
1246                         logger.info("creating new Taxon from TaxonName '" + nonViralName.getTitleCache()+"'");
1247                         taxon = Taxon.NewInstance(nonViralName, sec);
1248                 }
1249
1250                 //citation
1251                 Element elCitation = elScope.getChild("Citation",sddNamespace);
1252                 if (elCitation != null) {
1253                         String refCitation = elCitation.getAttributeValue("ref");
1254                         if (! refCitation.equals("")){
1255                                 citations.put(idCD, refCitation);
1256                         }
1257                         String location = elCitation.getAttributeValue("location");
1258                         if (! location.equals("")){
1259                                 locations.put(idCD, location);
1260                         }
1261                 }
1262                 return taxon;
1263         }
1264
1265         /**
1266          * @param sddNamespace
1267          * @param taxonDescription
1268          * @param elSummaryData
1269          */
1270         private void handleSummaryTextData(Namespace sddNamespace,
1271                         TaxonDescription taxonDescription, Element elSummaryData) {
1272                 String ref;
1273                 int k;
1274                 // <TextChar ref="c3">
1275                 List<Element> elTextChars = elSummaryData.getChildren("TextChar", sddNamespace);
1276                 k = 0;
1277                 //for each TextChar
1278                 for (Element elTextChar : elTextChars){
1279                         if ((++k % modCount) == 0){ logger.info("TextChar handled: " + (k-1));}
1280                         ref = elTextChar.getAttributeValue("ref");
1281                         Feature feature = features.get(ref);
1282                         TextData textData = TextData.NewInstance();
1283                         textData.setFeature(feature);
1284
1285                         // <Content>Free form text</Content>
1286                         String content = (String)ImportHelper.getXmlInputValue(elTextChar, "Content",sddNamespace);
1287                         textData.putText(datasetLanguage, content);
1288                         taxonDescription.addElement(textData);
1289                 }
1290         }
1291
1292         /**
1293          * @param sddNamespace
1294          * @param taxonDescription
1295          * @param elSummaryData
1296          */
1297         private void handleSummaryQuantitativeData(Namespace sddNamespace,
1298                         TaxonDescription taxonDescription, Element elSummaryData) {
1299                 String ref;
1300                 int k;
1301                 // <Quantitative ref="c2">
1302                 List<Element> elQuantitatives = elSummaryData.getChildren("Quantitative", sddNamespace);
1303                 k = 0;
1304                 //for each Quantitative
1305                 for (Element elQuantitative : elQuantitatives){
1306                         if ((++k % modCount) == 0){ logger.warn("Quantitative handled: " + (k-1));}
1307                         ref = elQuantitative.getAttributeValue("ref");
1308                         Feature feature = features.get(ref);
1309                         QuantitativeData quantitativeData = QuantitativeData.NewInstance();
1310                         quantitativeData.setFeature(feature);
1311
1312                         MeasurementUnit unit = units.get(ref);
1313                         String prefix = defaultUnitPrefixes.get(ref);
1314                         if (unit != null) {
1315                                 String u = unit.getLabel();
1316                                 if (prefix != null) {
1317                                         u = prefix + u;
1318                                 }
1319                                 unit.setLabel(u);
1320                                 quantitativeData.setUnit(unit);
1321                         }
1322
1323                         // <Measure type="Min" value="2.3"/>
1324                         List<Element> elMeasures = elQuantitative.getChildren("Measure", sddNamespace);
1325                         int l = 0;
1326
1327                         //for each State
1328                         for (Element elMeasure : elMeasures){
1329                                 if ((++l % modCount) == 0){ logger.info("States handled: " + (l-1));}
1330                                 String type = elMeasure.getAttributeValue("type");
1331                                 String value = elMeasure.getAttributeValue("value");
1332                                 if (value.contains(",")) {
1333                                         value = value.replace(',', '.');
1334                                 }
1335                                 Float v = Float.parseFloat(value);
1336                                 //Float v = new Float(0);
1337                                 StatisticalMeasure t = null;
1338                                 if (type.equals("Min")) {
1339                                         t = StatisticalMeasure.MIN();
1340                                 } else if (type.equals("Mean")) {
1341                                         t = StatisticalMeasure.AVERAGE();
1342                                 } else if (type.equals("Max")) {
1343                                         t = StatisticalMeasure.MAX();
1344                                 } else if (type.equals("SD")) {
1345                                         t = StatisticalMeasure.STANDARD_DEVIATION();
1346                                 } else if (type.equals("N")) {
1347                                         t = StatisticalMeasure.SAMPLE_SIZE();
1348                                 } else if (type.equals("UMethLower")) {
1349                                         t = StatisticalMeasure.TYPICAL_LOWER_BOUNDARY();
1350                                 } else if (type.equals("UMethUpper")) {
1351                                         t = StatisticalMeasure.TYPICAL_UPPER_BOUNDARY();
1352                                 } else if (type.equals("Var")) {
1353                                         t = StatisticalMeasure.VARIANCE();
1354                                 } else {
1355                                         t = StatisticalMeasure.NewInstance(type,type,type);
1356                                         statisticalMeasures.add(t);
1357                                 }
1358
1359                                 StatisticalMeasurementValue statisticalValue = StatisticalMeasurementValue.NewInstance();
1360                                 statisticalValue.setValue(v);
1361                                 statisticalValue.setType(t);
1362                                 quantitativeData.addStatisticalValue(statisticalValue);
1363                                 featureData.add(statisticalValue);
1364                         }
1365                         taxonDescription.addElement(quantitativeData);
1366                 }
1367         }
1368
1369         /**
1370          * @param sddNamespace
1371          * @param taxonDescription
1372          * @param elSummaryData
1373          */
1374         private void handleSummaryCategoricalData(Namespace sddNamespace,
1375                         TaxonDescription taxonDescription, Element elSummaryData) {
1376                 String ref;
1377                 // <Categorical ref="c4">
1378                 List<Element> elCategoricals = elSummaryData.getChildren("Categorical", sddNamespace);
1379                 int k = 0;
1380                 //for each Categorical
1381                 for (Element elCategorical : elCategoricals){
1382                         if ((++k % modCount) == 0){ logger.warn("Categorical handled: " + (k-1));}
1383                         ref = elCategorical.getAttributeValue("ref");
1384                         Feature feature = features.get(ref);
1385                         CategoricalData categoricalData = CategoricalData.NewInstance();
1386                         categoricalData.setFeature(feature);
1387
1388                         // <State ref="s3"/>
1389                         List<Element> elStates = elCategorical.getChildren("State", sddNamespace);
1390                         int l = 0;
1391
1392                         //for each State
1393                         for (Element elState : elStates){
1394                                 if ((++l % modCount) == 0){ logger.info("States handled: " + (l-1));}
1395                                 ref = elState.getAttributeValue("ref");
1396                                 State state = states.get(ref);
1397                                 if (state != null) {
1398                                         StateData stateData = StateData.NewInstance();
1399                                         stateData.setState(state);
1400                                         List<Element> elModifiers = elState.getChildren("Modifier", sddNamespace);
1401                                         for (Element elModifier : elModifiers){
1402                                                 ref = elModifier.getAttributeValue("ref");
1403                                                 DefinedTerm modifier = modifiers.get(ref);
1404                                                 if (modifier != null) {
1405                                                         stateData.addModifier(modifier);
1406                                                 }
1407                                         }
1408                                         categoricalData.addState(stateData);
1409                                 }
1410                                 taxonDescription.addElement(categoricalData);
1411                         }
1412                 }
1413         }
1414
1415         // imports the persons associated with the dataset creation, modification, related publications
1416         protected void importAgents(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1417                 // <Agents>
1418                 logger.info("start Agents ...");
1419                 Element elAgents = elDataset.getChild("Agents",sddNamespace);
1420                 if (elAgents != null) {
1421                         // <Agent id="a1">
1422                         List <Element> listAgents = elAgents.getChildren("Agent", sddNamespace);
1423                         int j = 0;
1424                         //for each Agent
1425                         for (Element elAgent : listAgents){
1426
1427                                 try {
1428
1429                                         String idA = elAgent.getAttributeValue("id");
1430
1431                                         //  <Representation>
1432                                         //   <Label>Kevin Thiele</Label>
1433                                         //   <Detail role="Description">Ali Baba is also known as r.a.m.</Detail>
1434                                         //  </Representation>
1435                                         Person person = Person.NewInstance();
1436                                         importRepresentation(elAgent, sddNamespace, person, idA, cdmState);
1437                                         person.addSource(IdentifiableSource.NewDataImportInstance(idA, "Agent"));
1438
1439                                         /*XIM <Links>
1440                                         Element elLinks = elAgent.getChild("Links",sddNamespace);
1441
1442                                         if (elLinks != null) {
1443
1444                                                 //  <Link rel="Alternate" href="http://www.diversitycampus.net/people/hagedorn"/>
1445                                                 List<Element> listLinks = elLinks.getChildren("Link", sddNamespace);
1446                                                 int k = 0;
1447                                                 //for each Link
1448                                                 for (Element elLink : listLinks){
1449
1450                                                         try {
1451
1452                                                                 String rel = elLink.getAttributeValue("rel");
1453                                                                 String href = elLink.getAttributeValue("href");
1454
1455                                                                 Media link = Media.NewInstance();
1456                                                                 MediaRepresentation mr = MediaRepresentation.NewInstance();
1457                                                                 mr.addRepresentationPart(MediaRepresentationPart.NewInstance(href, null));
1458                                                                 link.addRepresentation(mr);
1459                                                                 person.addMedia(link);
1460
1461                                                         } catch (Exception e) {
1462                                                                 //FIXME
1463                                                                 logger.warn("Import of Link " + k + " failed.");
1464                                                                 success = false;
1465                                                         }
1466
1467                                                         if ((++k % modCount) == 0){ logger.info("Links handled: " + k);}
1468
1469                                                 }
1470                                         }
1471                                         */
1472                                         if (authors.containsKey(idA)) {
1473                                                 authors.put(idA,person);
1474                                         }
1475
1476                                         if (editors.containsKey(idA)) {
1477                                                 editors.put(idA, person);
1478                                         }
1479
1480                                 } catch (Exception e) {
1481                                         //FIXME
1482                                         logger.warn("Import of Agent " + j + " failed.");
1483                                         cdmState.setUnsuccessfull();
1484                                 }
1485
1486                                 if ((++j % modCount) == 0){ logger.info("Agents handled: " + j);}
1487
1488                         }
1489                 }
1490         }
1491
1492         // imports publications related with the data set
1493         protected void importPublications(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1494                 /* <Publications>
1495                           <Publication id="p112">
1496                             <Representation>
1497                               <Label>Gee, X. & Haa, Y. (2003). How to be happy in five minutes. Instant Gratifications, Palm Beach.</Label>
1498                             </Representation>
1499                             <Links>
1500                             <Link rel="BasedOn" href="doi:10.1992/32311"/>
1501                             <Link rel="Alternate" href="http://some.service.net/providing/bibliographic.data"/>
1502                             </Links>
1503                         </Publications>
1504 */
1505                 logger.info("start Publications ...");
1506                 Element elPublications = elDataset.getChild("Publications",sddNamespace);
1507
1508                 if (elPublications != null) {
1509                         List<Element> listPublications = elPublications.getChildren("Publication", sddNamespace);
1510                         int j = 0;
1511                         for (Element elPublication : listPublications){
1512
1513                                 try {
1514
1515                                         String idP = elPublication.getAttributeValue("id");
1516                                         Reference<?> publication = ReferenceFactory.newArticle();
1517                                         importRepresentation(elPublication, sddNamespace, publication, idP, cdmState);
1518
1519                                         publications.put(idP,publication);
1520
1521                                 } catch (Exception e) {
1522                                         logger.warn("Import of Publication " + j + " failed.");
1523                                         cdmState.setUnsuccessfull();
1524                                 }
1525
1526                                 if ((++j % modCount) == 0){ logger.info("Publications handled: " + j);}
1527
1528                         }
1529                 }
1530         }
1531
1532         // imports media objects such as images //FIXME check mediaobj
1533         protected void importMediaObjects(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1534                 // <MediaObjects>
1535                 logger.info("start MediaObjects ...");
1536                 Element elMediaObjects = elDataset.getChild("MediaObjects",sddNamespace);
1537
1538                 if (elMediaObjects != null) {
1539                         // <MediaObject id="m1">
1540                         List<Element> listMediaObjects = elMediaObjects.getChildren("MediaObject", sddNamespace);
1541                         int j = 0;
1542                         for (Element elMO : listMediaObjects){
1543
1544                                 String id = "";
1545
1546                                 try {
1547                                         String idMO = elMO.getAttributeValue("id");
1548                                         id = idMO;
1549
1550                                         //  <Representation>
1551                                         //   <Label>Image description, e.g. to be used for alt-attribute in html.</Label>
1552                                         //  </Representation>
1553                                         Media media = Media.NewInstance();
1554                                         importRepresentation(elMO, sddNamespace, media, idMO, cdmState);
1555
1556                                         // <Type>Image</Type>
1557                                         // <Source href="http://test.edu/test.jpg"/>
1558                                         String type = (String)ImportHelper.getXmlInputValue(elMO,"Type",sddNamespace);
1559
1560                                         if ((type != null) && (type.equals("Image"))) {
1561                                                 Element elSource = elMO.getChild("Source",sddNamespace);
1562                                                 String href = elSource.getAttributeValue("href");
1563
1564                                                 ImageInfo imageMetaData = null;
1565                                                 ImageFile image = null;
1566
1567                                                 if (href.substring(0,7).equals("http://")) {
1568                                                         try{
1569                                                                 URL url = new URL(href);
1570
1571                                                                 imageMetaData = ImageInfo.NewInstance(url.toURI(), 0);
1572                                                                 image = ImageFile.NewInstance(url.toURI(), null, imageMetaData);
1573                                                         } catch (MalformedURLException e) {
1574                                                                 logger.error("Malformed URL", e);
1575                                                         } catch (IOException ioe) {
1576                                                                 logger.warn("(IO ex: " + id + "): " + ioe.getMessage());
1577
1578                                                         }
1579                                                 } else {
1580                                                         String sns = cdmState.getConfig().getSourceNameString();
1581                                                         File f = new File(sns);
1582                                                         File parent = f.getParentFile();
1583                                                         String fi = parent.toString() + File.separator + href;
1584                                                         File file = new File(fi);
1585                                                         imageMetaData  = ImageInfo.NewInstance(new URI(fi), 0); //file
1586                                                         image = ImageFile.NewInstance(file.toURI(), null, imageMetaData);
1587                                                 }
1588                                                 MediaRepresentation representation = MediaRepresentation.NewInstance(imageMetaData.getMimeType(), null);
1589                                                 representation.addRepresentationPart(image);
1590
1591                                                 media.addRepresentation(representation);
1592
1593                                                 ArrayList<CdmBase> lcb = (ArrayList<CdmBase>) mediaObject_ListCdmBase.get(idMO);
1594                                                 if (lcb != null) {
1595                                                         for (int k = 0; k < lcb.size(); k++) {
1596                                                                 if (lcb.get(k) instanceof DefinedTermBase) {
1597                                                                         DefinedTermBase<?> dtb = (DefinedTermBase<?>) lcb.get(k);
1598                                                                         // if (lcb.get(0) instanceof DefinedTermBase) {
1599                                                                         // DefinedTermBase dtb = (DefinedTermBase) lcb.get(0);
1600                                                                         //                                                                      if (dtb!=null) {
1601                                                                         //                                                                              if (k == 0) {
1602                                                                         dtb.addMedia(media);
1603                                                                         //System.out.println(dtb.getLabel());
1604                                                                         //                                                                              } else {
1605                                                                         //                                                                                      Media me = (Media) media.clone();
1606                                                                         //                                                                                      dtb.addMedia(me);
1607                                                                         //                                                                              }
1608                                                                         //                                                                      }
1609                                                                 } else if (lcb.get(k) instanceof Reference) {
1610                                                                         Reference<?> rb = (Reference<?>) lcb.get(k);
1611                                                                         //} else if (lcb.get(0) instanceof Reference) {
1612                                                                         //Reference rb = (Reference) lcb.get(0);
1613                                                                         // rb.setTitleCache(label);
1614                                                                         //                                                                      if (rb!=null) {
1615                                                                         //                                                                              if (k == 0) {
1616                                                                         rb.addMedia(media);
1617                                                                         //System.out.println(rb.getTitle());
1618                                                                         //                                                                              } else {
1619                                                                         //                                                                                      Media me = (Media) media.clone();
1620                                                                         //                                                                                      rb.addMedia(me);
1621                                                                         //                                                                              }
1622                                                                         //                                                                      }
1623 //                                                               else if (lcb.get(k) instanceof TaxonNameBase){
1624 //                                                                      TaxonNameBase tb = (TaxonNameBase) lcb.get(k);
1625 //                                                                      tb.addMedia(media);
1626                                                                 } else {
1627                                                                         logger.warn("Can't handle associated media for " + lcb.get(k).getId() + "(" +  lcb.get(k).getClass().getSimpleName()+")"  );
1628                                                                 }
1629                                                         }
1630                                                 }
1631                                         }
1632
1633                                 } catch (Exception e) {
1634                                         //FIXME
1635                                         logger.warn("Could not attach MediaObject " + j + "(SDD: " + id + ") to several objects: " + e.getMessage());
1636                                         cdmState.setUnsuccessfull();
1637                                 }
1638
1639                                 if ((++j % modCount) == 0){ logger.info("MediaObjects handled: " + j);
1640
1641                                 }
1642                         }
1643                 }
1644         }
1645
1646         // imports the <DescriptiveConcepts> block ; DescriptiveConcepts are used as nodes in CharacterTrees and Characters as leaves
1647         // but since Modifiers can be linked to DescriptiveConcepts they are stored as features with a particular Marker
1648         protected void importDescriptiveConcepts(Element elDataset, Namespace sddNamespace, SDDImportState state){
1649                 /* <DescriptiveConcepts>
1650                       <DescriptiveConcept id="dc0">
1651                                 <Representation>
1652                                   <Label>Fixed set of modifiers supported in Lucid3</Label>
1653                                 </Representation>
1654                                 <Modifiers>
1655                                   <Modifier id="mod1">
1656                                     <Representation>
1657                                       <Label>rarely</Label>
1658                                     </Representation>
1659                                     <ModifierClass>Frequency</ModifierClass>
1660                                     <ProportionRange lowerestimate="0.0" upperestimate="0.25"/>
1661                                   </Modifier>
1662                           </Modifiers>
1663                         </DescriptiveConcept>
1664                  </DescriptiveConcepts>
1665                  */
1666                 logger.info("start DescriptiveConcepts ...");
1667                 Element elDescriptiveConcepts = elDataset.getChild("DescriptiveConcepts",sddNamespace);
1668                 if (elDescriptiveConcepts != null) {
1669                         List<Element> listDescriptiveConcepts = elDescriptiveConcepts.getChildren("DescriptiveConcept", sddNamespace);
1670                         int j = 0;
1671
1672                         for (Element elDescriptiveConcept : listDescriptiveConcepts){
1673                                 try {
1674                                 String id = elDescriptiveConcept.getAttributeValue("id");
1675                                         Feature feature = Feature.NewInstance();
1676                                         feature.setKindOf(Feature.DESCRIPTION());
1677                                         if (!id.equals("")) {
1678                                         //       <Representation>
1679                                         //       <Label>Body</Label>
1680                                         importRepresentation(elDescriptiveConcept, sddNamespace, feature, id, state);
1681                                                 features.put(id, feature);
1682                                                 getTermService().save(feature);//XIM
1683                                                 descriptiveConcepts.add(feature);
1684                                                 // imports the modifiers
1685                                                 Element elModifiers = elDescriptiveConcept.getChild("Modifiers", sddNamespace);
1686                                         if (elModifiers !=null){
1687                                                 List<Element> listModifiers = elModifiers.getChildren("Modifier", sddNamespace);
1688                                                         TermVocabulary<DefinedTerm> termVocabularyState = TermVocabulary.NewInstance(TermType.Modifier, null, null, null, null);
1689                                                 for (Element elModifier : listModifiers) {
1690                                                                 DefinedTerm modif = DefinedTerm.NewModifierInstance(null, null, null);
1691                                                                 String idmod = elModifier.getAttributeValue("id");
1692                                                                 importRepresentation(elModifier, sddNamespace, modif, idmod, state);
1693                                                                 termVocabularyState.addTerm(modif);
1694                                                                 //termVocabularyStates.add(termVocabularyState);
1695                                                                 getVocabularyService().save(termVocabularyState);//XIM
1696                                                                 modifiers.put(idmod, modif);
1697                                                 }
1698                                                         feature.addRecommendedModifierEnumeration(termVocabularyState);
1699                                 }
1700
1701                                         }
1702                                 }
1703                                 catch (Exception e) {
1704                                         logger.warn("Import of DescriptiveConcept " + j + " failed: " + e.getMessage());
1705                                 }
1706                                 if ((++j % modCount) == 0){ logger.info("DescriptiveConcepts handled: " + j);}
1707
1708                         }
1709                 }
1710         }
1711
1712         // imports the <CharacterTrees> block
1713         protected void importCharacterTrees(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1714                 // <CharacterTrees>
1715                 logger.info("start CharacterTrees ...");
1716                 Element elCharacterTrees = elDataset.getChild("CharacterTrees",sddNamespace);
1717
1718                 if (elCharacterTrees != null) {
1719                         List<Element> listCharacterTrees = elCharacterTrees.getChildren("CharacterTree", sddNamespace);
1720                         int j = 0;
1721                         for (Element elCharacterTree : listCharacterTrees){
1722                                 try {
1723                                         Element elRepresentation = elCharacterTree.getChild("Representation",sddNamespace);
1724                                         String label = (String)ImportHelper.getXmlInputValue(elRepresentation,"Label",sddNamespace);
1725                                         //Element elDesignedFor = elCharacterTree.getChild("DesignedFor",sddNamespace);//TODO ?
1726
1727                                         FeatureTree featureTree =  FeatureTree.NewInstance();
1728                                         importRepresentation(elCharacterTree, sddNamespace, featureTree, "", cdmState);
1729                                         FeatureNode root = featureTree.getRoot();
1730                                         List<Element> listeOfNodes = elCharacterTree.getChildren("Nodes", sddNamespace);
1731
1732                                         //Nodes of CharacterTrees in SDD always refer to DescriptiveConcepts
1733                                         for (Element elNodes : listeOfNodes) {
1734                                                 handleCharacterNodes(sddNamespace, root, elNodes);
1735                                         }
1736                                         featureTrees.add(featureTree);
1737                                         if (workingSet.getDescriptiveSystem() != null){
1738                                                 //TODO how to handle multiple
1739                                                 logger.warn("Multiple feature trees not yet supported");
1740                                         }else{
1741                                                 workingSet.setDescriptiveSystem(featureTree);
1742                                         }
1743                                 }
1744
1745                                 catch (Exception e) {
1746                                         logger.warn("Import of Character tree " + j + " failed.");
1747                                         cdmState.setUnsuccessfull();
1748                                 }
1749                                 if ((++j % modCount) == 0){ logger.info("CharacterTrees handled: " + j);}
1750
1751                         }
1752
1753                 }
1754         }
1755
1756         /**
1757          * @param sddNamespace
1758          * @param root
1759          * @param elNodes
1760          */
1761         private void handleCharacterNodes(Namespace sddNamespace, FeatureNode root, Element elNodes) {
1762                 List<Element> listNodes = elNodes.getChildren("Node", sddNamespace);
1763                 if (listNodes != null) {
1764                         for (Element elNode : listNodes){
1765                                 String idN = elNode.getAttributeValue("id");
1766                                 FeatureNode fn = null;
1767                                 Feature dc = null;
1768                                 if (idN!=null) {
1769                                         // DescriptiveConcepts are used as nodes in CharacterTrees
1770                                         Element elDescriptiveConcept = elNode.getChild("DescriptiveConcept", sddNamespace);
1771                                         if (elDescriptiveConcept != null){
1772                                                 String refDC = elDescriptiveConcept.getAttributeValue("ref");
1773                                                 dc = features.get(refDC);
1774                                                 fn = FeatureNode.NewInstance(dc);
1775                                         }
1776                                         if (fn==null){
1777                                                 fn = FeatureNode.NewInstance();
1778                                         }
1779                                         Element elParent = elNode.getChild("Parent", sddNamespace);
1780                                         // in SDD links between Nodes are referenced by the <Parent> tag
1781                                         if (elParent!=null){
1782                                                 String refP = elParent.getAttributeValue("ref");
1783                                                 if (refP!=null) {
1784                                                         FeatureNode parent = featureNodes.get(refP);
1785                                                         if (parent==null){
1786                                                                 root.addChild(fn); // if no parent found or the reference is broken, add the node to the root of the tree
1787                                                         }
1788                                                         else {
1789                                                                 parent.addChild(fn);
1790                                                         }
1791                                                 }
1792                                         }
1793                                         else {
1794                                                 root.addChild(fn); // if no parent found or the reference is broken, add the node to the root of the tree
1795                                         }
1796                                 }
1797                                 featureNodes.put(idN, fn);
1798                         }
1799                 }
1800
1801                 // Leaves of CharacterTrees in SDD are always CharNodes (referring to Characters)
1802                 List<Element> listCharNodes = elNodes.getChildren("CharNode", sddNamespace);
1803                 if (listCharNodes != null) {
1804                         for (Element elCharNode : listCharNodes){
1805                                 Element elParent = elCharNode.getChild("Parent", sddNamespace);
1806                                 Element elCharacter = elCharNode.getChild("Character", sddNamespace);
1807                                 Element elDependencyRules = elCharNode.getChild("DependencyRules", sddNamespace);
1808                                 FeatureNode fn = FeatureNode.NewInstance();
1809
1810                                 if (elDependencyRules!=null){
1811                                         Element elInapplicableIf = elCharNode.getChild("InapplicableIf", sddNamespace);
1812                                         if (elInapplicableIf!=null){
1813                                                 List<Element> listStates = elInapplicableIf.getChildren("State", sddNamespace);
1814                                                 for (Element stateElement : listStates) {
1815                                                         String refState = stateElement.getAttributeValue("ref");
1816                                                         if ((refState!=null)&&(!refState.equals(""))) {
1817                                                                 State state = states.get(refState);
1818                                                                 fn.addInapplicableState(state);
1819                                                         }
1820                                                 }
1821                                         }
1822                                         Element elOnlyapplicableIf = elCharNode.getChild("OnlyApplicableIf", sddNamespace);
1823                                         if (elOnlyapplicableIf!=null){
1824                                                 List<Element> listStates = elInapplicableIf.getChildren("State", sddNamespace);
1825                                                 for (Element stateElement : listStates) {
1826                                                         String refState = stateElement.getAttributeValue("ref");
1827                                                         if ((refState!=null)&&(!refState.equals(""))) {
1828                                                                 State state = states.get(refState);
1829                                                                 fn.addApplicableState(state);
1830                                                         }
1831                                                 }
1832                                         }
1833                                 }
1834
1835                                 if (elParent!=null){
1836                                         String refP = elParent.getAttributeValue("ref");
1837                                         if ((refP!=null)&&(!refP.equals(""))) {
1838                                         FeatureNode parent = featureNodes.get(refP);
1839                                                 if (parent==null){
1840                                                 parent = root; // if no parent found or the reference is broken, add the node to the root of the tree
1841                                                 }
1842                                                 parent.addChild(fn);
1843                                         }
1844                                 }
1845                                 String refC = elCharacter.getAttributeValue("ref");
1846                                 if ((refC!=null)&&(!refC.equals(""))){
1847                                         Feature character = features.get(refC);
1848                                         fn.setFeature(character);
1849                                         featureNodes.put(refC, fn);
1850                                 }
1851                         }
1852                 }
1853         }
1854
1855         // imports the <TaxonHierarchies> block
1856         protected void importTaxonHierarchies(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1857
1858                 logger.info("start TaxonHierarchies ...");
1859                 Element elTaxonHierarchies = elDataset.getChild("TaxonHierarchies",sddNamespace);
1860
1861                 if (elTaxonHierarchies != null) {
1862                         List<Element> listTaxonHierarchies = elTaxonHierarchies.getChildren("TaxonHierarchy", sddNamespace);
1863                         int j = 0;
1864                         for (Element elTaxonHierarchy : listTaxonHierarchies){
1865                                 try {
1866                                         Element elRepresentation = elTaxonHierarchy.getChild("Representation",sddNamespace);
1867                                         String label = (String)ImportHelper.getXmlInputValue(elRepresentation,"Label",sddNamespace);
1868                                                 Classification classification =  Classification.NewInstance(label);
1869                                                 importRepresentation(elTaxonHierarchy, sddNamespace, classification, "", cdmState);
1870
1871                                                 Element elNodes = elTaxonHierarchy.getChild("Nodes", sddNamespace); // There can be only one <Nodes> block for TaxonHierarchies
1872                                                 List<Element> listNodes = elNodes.getChildren("Node", sddNamespace);
1873
1874                                                 for (Element elNode : listNodes){
1875                                                         String idN = elNode.getAttributeValue("id");
1876                                                         TaxonNameBase<?,?> tnb = null;
1877                                                         if (!idN.equals("")) {
1878                                                                 Element elTaxonName = elNode.getChild("TaxonName", sddNamespace);
1879                                                                 String refTN = elTaxonName.getAttributeValue("ref");
1880                                                                 tnb = taxonNameBases.get(refTN);
1881                                                                 Taxon taxon = (Taxon) tnb.getTaxa().iterator().next() ;
1882                                                                 Element elParent = elNode.getChild("Parent", sddNamespace);
1883                                                                 if (elParent!=null){
1884                                                                         String refP = elParent.getAttributeValue("ref");
1885                                                                         if (!refP.equals("")) {
1886                                                                                 TaxonNode parent = taxonNodes.get(refP);
1887                                                                                 TaxonNode child = parent.addChildTaxon(taxon, sec, "", Synonym.NewInstance(tnb, sec));
1888                                                                                 taxonNodes.put(idN,child);
1889                                                                         }
1890                                                                 }
1891                                                                 else {
1892                                                                         TaxonNode tn = classification.addChildTaxon(taxon, sec, "", Synonym.NewInstance(tnb, sec)); // if no parent found or the reference is broken, add the node to the root of the tree
1893                                                                         taxonNodes.put(idN,tn);
1894                                                                 }
1895                                                         }
1896                                                 }
1897
1898                                                 classifications.add(classification);
1899                                         }
1900
1901                                 catch (Exception e) {
1902                                         //FIXME
1903                                         logger.warn("Import of Taxon Hierarchy " + j + " failed.");
1904                                         cdmState.setUnsuccessfull();
1905                                 }
1906
1907                                 if ((++j % modCount) == 0){ logger.info("TaxonHierarchies handled: " + j);}
1908
1909                         }
1910
1911                 }
1912         }
1913
1914
1915         // imports the <GeographicAreas> block
1916         protected void importGeographicAreas(Element elDataset, Namespace sddNamespace, SDDImportState cdmState) {
1917                 Element elGeographicAreas = elDataset.getChild("GeographicAreas",sddNamespace);
1918                 if (elGeographicAreas != null) {
1919                         List<Element> listGeographicAreas = elGeographicAreas.getChildren("GeographicArea", sddNamespace);
1920                         int j = 0;
1921
1922                         for (Element elGeographicArea : listGeographicAreas){
1923
1924                                 String id = elGeographicArea.getAttributeValue("id");
1925                                 NamedArea na = new NamedArea();
1926                                 importRepresentation(elGeographicArea, sddNamespace, na, id, cdmState);
1927                                 namedAreas.put(id,na);
1928                                                                 }
1929                         if ((++j % modCount) == 0){ logger.info("GeographicAreas handled: " + j);}
1930                                                         }
1931                                                         }
1932 }