cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/sdd/in/SDDDataSetImport.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.sdd.in;
  11
  12 import java.io.File;
  13 import java.net.MalformedURLException;
  14 import java.net.URI;
  15 import java.net.URL;
  16 import java.text.SimpleDateFormat;
  17 import java.util.ArrayList;
  18 import java.util.Date;
  19 import java.util.HashMap;
  20 import java.util.HashSet;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Set;
  24
  25 import org.apache.commons.lang.StringUtils;
  26 import org.apache.log4j.Logger;
  27 import org.jdom.Element;
  28 import org.jdom.Namespace;
  29 import org.joda.time.DateTime;
  30 import org.springframework.stereotype.Component;
  31 import org.springframework.transaction.TransactionStatus;
  32
  33 import eu.etaxonomy.cdm.api.service.IDescriptionService;
  34 import eu.etaxonomy.cdm.common.media.ImageInfo;
  35 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  36 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  37 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  38 import eu.etaxonomy.cdm.io.common.ICdmIO;
  39 import eu.etaxonomy.cdm.io.common.ICdmImport;
  40 import eu.etaxonomy.cdm.io.common.ImportHelper;
  41 import eu.etaxonomy.cdm.io.sdd.SDDTransformer;
  42 import eu.etaxonomy.cdm.model.agent.Person;
  43 import eu.etaxonomy.cdm.model.agent.Team;
  44 import eu.etaxonomy.cdm.model.common.Annotation;
  45 import eu.etaxonomy.cdm.model.common.AnnotationType;
  46 import eu.etaxonomy.cdm.model.common.CdmBase;
  47 import eu.etaxonomy.cdm.model.common.DefinedTerm;
  48 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  49 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  50 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  51 import eu.etaxonomy.cdm.model.common.Language;
  52 import eu.etaxonomy.cdm.model.common.LanguageString;
  53 import eu.etaxonomy.cdm.model.common.Marker;
  54 import eu.etaxonomy.cdm.model.common.MarkerType;
  55 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  56 import eu.etaxonomy.cdm.model.common.Representation;
  57 import eu.etaxonomy.cdm.model.common.TermBase;
  58 import eu.etaxonomy.cdm.model.common.TermType;
  59 import eu.etaxonomy.cdm.model.common.TermVocabulary;
  60 import eu.etaxonomy.cdm.model.common.VersionableEntity;
  61 import eu.etaxonomy.cdm.model.description.CategoricalData;
  62 import eu.etaxonomy.cdm.model.description.Feature;
  63 import eu.etaxonomy.cdm.model.description.FeatureNode;
  64 import eu.etaxonomy.cdm.model.description.FeatureTree;
  65 import eu.etaxonomy.cdm.model.description.MeasurementUnit;
  66 import eu.etaxonomy.cdm.model.description.QuantitativeData;
  67 import eu.etaxonomy.cdm.model.description.State;
  68 import eu.etaxonomy.cdm.model.description.StateData;
  69 import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
  70 import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
  71 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  72 import eu.etaxonomy.cdm.model.description.TextData;
  73 import eu.etaxonomy.cdm.model.description.WorkingSet;
  74 import eu.etaxonomy.cdm.model.location.NamedArea;
  75 import eu.etaxonomy.cdm.model.media.IdentifiableMediaEntity;
  76 import eu.etaxonomy.cdm.model.media.ImageFile;
  77 import eu.etaxonomy.cdm.model.media.Media;
  78 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
  79 import eu.etaxonomy.cdm.model.media.MediaRepresentationPart;
  80 import eu.etaxonomy.cdm.model.media.Rights;
  81 import eu.etaxonomy.cdm.model.name.NonViralName;
  82 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  83 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  84 import eu.etaxonomy.cdm.model.reference.Reference;
  85 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  86 import eu.etaxonomy.cdm.model.taxon.Classification;
  87 import eu.etaxonomy.cdm.model.taxon.Synonym;
  88 import eu.etaxonomy.cdm.model.taxon.Taxon;
  89 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  90
  91 /**
  92  * This class is not used yet. It is meant for refactoring the SDD import.
  93  *
  94  * @author a.mueller
  95  * @created 06.01.2011
  96  * @version 1.0
  97  */
  98 @Component
  99 public class SDDDataSetImport extends CdmImportBase<SDDImportConfigurator, SDDImportState> implements ICdmImport<SDDImportConfigurator, SDDImportState> {
 100         private static final Logger logger = Logger.getLogger(SDDDataSetImport.class);
 101
 102         private static int modCount = 1000;
 103
 104         private Map<String,Person> authors = new HashMap<String,Person>();
 105         private Map<String,String> citations = new HashMap<String,String>();
 106         private Map<String,String> defaultUnitPrefixes = new HashMap<String,String>();
 107         private Map<String,Person> editors = new HashMap<String,Person>();
 108         private Map<String,FeatureNode> featureNodes = new HashMap<String,FeatureNode>();
 109         private Map<String,Feature> features = new HashMap<String,Feature>();
 110         private Map<String,String> locations = new HashMap<String,String>();
 111         private Map<String,List<CdmBase>> mediaObject_ListCdmBase = new HashMap<String,List<CdmBase>>();
 112         private Map<String,String> mediaObject_Role = new HashMap<String,String>();
 113         private Map<String,Reference> publications = new HashMap<String,Reference>();
 114         private Map<String,State> states = new HashMap<String,State>();
 115         private Map<String,TaxonDescription> taxonDescriptions = new HashMap<String,TaxonDescription>();
 116         private Map<String,NonViralName> taxonNameBases = new HashMap<String,NonViralName>();
 117         private Map<String,MeasurementUnit> units = new HashMap<String,MeasurementUnit>();
 118         private Map<String,TaxonNode> taxonNodes = new HashMap<String,TaxonNode>();
 119         private Map<String,NamedArea> namedAreas = new HashMap<String,NamedArea>();
 120         private Map<String,DerivedUnit> specimens = new HashMap<String,DerivedUnit>();
 121         private Map<String,DefinedTerm> modifiers = new HashMap<String,DefinedTerm>();
 122
 123         private Set<MarkerType> markerTypes = new HashSet<MarkerType>();
 124         private Set<TermVocabulary> vocabularies = new HashSet<TermVocabulary>();
 125
 126         private Set<Feature> descriptiveConcepts = new HashSet<Feature>();
 127         private Set<AnnotationType> annotationTypes = new HashSet<AnnotationType>();
 128 //      private Set<Feature> featureSet = new HashSet<Feature>();
 129         private Set<Reference> sources = new HashSet<Reference>();
 130         private Reference sec = ReferenceFactory.newDatabase();
 131         private Reference sourceReference = null;
 132
 133         private Language datasetLanguage = null;
 134         private WorkingSet workingSet = null;
 135
 136         private Namespace xmlNamespace = Namespace.getNamespace("xml","http://www.w3.org/XML/1998/namespace");
 137
 138         private String generatorName = "";
 139         private String generatorVersion = "";
 140
 141
 142         private Set<StatisticalMeasure> statisticalMeasures = new HashSet<StatisticalMeasure>();
 143         private Set<VersionableEntity> featureData = new HashSet<VersionableEntity>();
 144         private Set<FeatureTree> featureTrees = new HashSet<FeatureTree>();
 145         private Set<Classification> classifications = new HashSet<Classification>();
 146
 147         private Rights copyright = null;
 148
 149         private int taxonNamesCount = 0;
 150
 151         protected Class<ICdmIO>[] ioClassList;
 152
 153         public SDDDataSetImport(){
 154                 super();
 155         }
 156
 157         protected void makeIoClassList(){
 158                 ioClassList = new Class[]{
 159                                 SDDImport.class
 160                 };
 161         };
 162
 163         @Override
 164         public boolean doCheck(SDDImportState state){
 165                 boolean result = true;
 166                 makeIoClassList();
 167                 logger.warn("No check implemented for SDD");
 168                 return result;
 169         }
 170
 171         //      @Override
 172         //      public boolean doInvoke(IImportConfigurator config, Map<String, MapWrapper<? extends CdmBase>> stores){
 173         @Override
 174         public void doInvoke(SDDImportState state){
 175
 176                 TransactionStatus ts = startTransaction();
 177                 SDDImportConfigurator sddConfig = state.getConfig();
 178                 IProgressMonitor progressMonitor = sddConfig.getProgressMonitor();
 179
 180                 logger.info("start Datasets ...");
 181
 182                 // <Datasets xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://rs.tdwg.org/UBIF/2006/" xsi:schemaLocation="http://rs.tdwg.org/UBIF/2006/ ../SDD.xsd">
 183                 Element root = sddConfig.getSourceRoot();
 184                 Namespace sddNamespace = sddConfig.getSddNamespace();
 185
 186                 logger.info("start TechnicalMetadata ...");
 187                 // <TechnicalMetadata created="2006-04-20T10:00:00">
 188                 importTechnicalMetadata(root, sddNamespace, sddConfig);
 189                 List<Element> elDatasets = root.getChildren("Dataset",sddNamespace);
 190                 int i = 0;
 191
 192                 //for each Dataset
 193                 logger.info("start Dataset ...");
 194                 progressMonitor.beginTask("Importing SDD data", elDatasets.size());
 195                 for (Element elDataset : elDatasets){
 196                         importDataset(elDataset, sddNamespace, state);
 197 //                      if ((++i % modCount) == 0){ logger.info("dataset(s) handled: " + i);}
 198 //                      logger.info(i + " dataset(s) handled");
 199                         progressMonitor.worked(1);
 200                 }
 201                 commitTransaction(ts);
 202                 progressMonitor.done();
 203                 logger.info("End of transaction");
 204                 return;
 205         }
 206
 207         /* (non-Javadoc)
 208          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 209          */
 210         protected boolean isIgnore(SDDImportState state){
 211                 return false;
 212         }
 213
 214
 215         // associates the reference of a media object in SDD with a CdmBase Object
 216         protected void associateImageWithCdmBase(String refMO, CdmBase cb){
 217                 if ((refMO != null) && (cb!=null)) {
 218                         if (! refMO.equals("")) {
 219                                 if (! mediaObject_ListCdmBase.containsKey(refMO)) {
 220                                         List<CdmBase> lcb = new ArrayList<CdmBase>();
 221                                         lcb.add(cb);
 222                                         mediaObject_ListCdmBase.put(refMO,lcb);
 223                                 } else {
 224                                         List<CdmBase> lcb = mediaObject_ListCdmBase.get(refMO);
 225                                         lcb.add(cb);
 226                                         mediaObject_ListCdmBase.put(refMO,lcb);
 227                                 }
 228                         }
 229                 }
 230         }
 231
 232         // imports information about the Dataset
 233         protected void importDatasetRepresentation(Element parent, Namespace sddNamespace){
 234                 logger.info("start Representation ...");
 235                 /* <Representation>
 236                         <Label>The Genus Viola</Label>
 237                         <Detail>This is an example for a very simple SDD file, representing a single description with categorical, quantitative, and text character. Compare also the "Fragment*" examples, which contain more complex examples in the form of document fragments. Intended for version="SDD 1.1".</Detail>
 238                </Representation>
 239                  */
 240
 241
 242
 243                 Element elRepresentation = parent.getChild("Representation",sddNamespace);
 244                 String label = (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace);
 245                 String detail = (String)ImportHelper.getXmlInputValue(elRepresentation, "Detail",sddNamespace);
 246
 247                 //new
 248                 Representation representation = Representation.NewInstance(detail, label, null, datasetLanguage);
 249                 workingSet.addRepresentation(representation);
 250
 251
 252                 //old
 253 //              sec.setTitleCache(label, true);
 254 //
 255 //              if (detail != null) {
 256 //                      Annotation annotation = Annotation.NewInstance(detail, datasetLanguage);
 257 //                      annotation.setAnnotationType(AnnotationType.EDITORIAL());
 258 //                      sec.addAnnotation(annotation);
 259 //              }
 260
 261
 262                 List<Element> listMediaObjects = elRepresentation.getChildren("MediaObject",sddNamespace);
 263
 264                 for (Element elMediaObject : listMediaObjects) {
 265                         String ref = null;
 266                         String role = null;
 267                         if (elMediaObject != null) {
 268                                 ref = elMediaObject.getAttributeValue("ref");
 269                                 role = elMediaObject.getAttributeValue("role");
 270                         }
 271                         if (ref != null) {
 272                                 if (!ref.equals("")) {
 273                                         this.associateImageWithCdmBase(ref,sourceReference);
 274                                         this.associateImageWithCdmBase(ref,sec);
 275                                         mediaObject_Role.put(ref,role);
 276                                 }
 277                         }
 278                 }
 279         }
 280
 281         // imports the representation (label, detail, lang) of a particular SDD element
 282         protected void importRepresentation(Element parent, Namespace sddNamespace, VersionableEntity ve, String id, SDDImportState state){
 283                 Element elRepresentation = parent.getChild("Representation",sddNamespace);
 284
 285                 Map<Language,List<String>> langLabDet = new HashMap<Language,List<String>>();
 286
 287                 handleRepresentationLabels(sddNamespace, elRepresentation, langLabDet);
 288                 handleRepresentationDetails(sddNamespace, elRepresentation, langLabDet);
 289
 290                 if (ve instanceof TermBase) {
 291                         makeRepresentationForTerms((TermBase)ve, langLabDet);
 292                 }else if (ve instanceof Media) {
 293                         makeRepresentationForMedia((Media)ve, langLabDet);
 294                 }else if (ve instanceof IdentifiableEntity<?>) {
 295                         IdentifiableEntity<?> ie = (IdentifiableEntity<?>)ve;
 296                         makeRepresentationForIdentifiableEntity(sddNamespace, ie, elRepresentation, langLabDet);
 297                         if (ve instanceof IdentifiableMediaEntity<?>){
 298                                 makeRepresentationForIdentifiableMediaEntity(parent, sddNamespace, (IdentifiableMediaEntity<?>)ve);
 299                         }
 300                 }
 301
 302                 makeRepresentationMediaObjects(sddNamespace, ve, elRepresentation);//FIXME
 303
 304         }
 305
 306
 307         /**
 308          * Handles the "Detail" children of representations. Adds the result to the langLabDet.
 309          * @param sddNamespace
 310          * @param elRepresentation
 311          * @param langLabDet
 312          */
 313         private void handleRepresentationDetails(Namespace sddNamespace,
 314                         Element elRepresentation, Map<Language, List<String>> langLabDet) {
 315                 List<Element> listDetails = elRepresentation.getChildren("Detail",sddNamespace);
 316                 for (Element elDetail : listDetails){
 317                         Language language = getLanguage(elDetail);
 318                         String role = elDetail.getAttributeValue("role");
 319                         String detail = elDetail.getText();
 320                         List<String> labDet = langLabDet.get(language);
 321                         labDet.add(detail);
 322                         labDet.add(role);
 323                         langLabDet.put(language, labDet);
 324                 }
 325         }
 326
 327         /**
 328          * Handles the "Label" children of representations. Adds the result to the langLabDet.
 329          * @param sddNamespace
 330          * @param elRepresentation
 331          * @param langLabDet
 332          */
 333         private void handleRepresentationLabels(Namespace sddNamespace,
 334                                 Element elRepresentation, Map<Language, List<String>> langLabDet) {
 335                 // <Label xml:lang="la">Viola hederacea Labill.</Label>
 336                 List<Element> listLabels = elRepresentation.getChildren("Label",sddNamespace);
 337                 for (Element elLabel : listLabels){
 338                         Language language = getLanguage(elLabel);
 339                         String label = elLabel.getText();
 340                         List<String> labDet = new ArrayList<String>(3);
 341                         labDet.add(label);
 342                         langLabDet.put(language, labDet);
 343                 }
 344         }
 345
 346         /**
 347          *
 348          * @param media
 349          * @param langLabDet
 350          */
 351         private void makeRepresentationForMedia(Media media, Map<Language, List<String>> langLabDet) {
 352                 for (Language lang : langLabDet.keySet()){
 353                         List<String> labDet = langLabDet.get(lang);
 354                         if (labDet.get(0) != null){
 355                                 media.putTitle(LanguageString.NewInstance(labDet.get(0), lang));
 356                         }
 357                         if (labDet.size()>1) {
 358                                 media.putDescription(lang, labDet.get(1));
 359                         }
 360                 }
 361         }
 362
 363         /**
 364          * Handles representations for terms. Adds one representation per language in langLabDet.
 365          *
 366          * @param ve
 367          * @param langLabDet
 368          */
 369         private void makeRepresentationForTerms(TermBase tb, Map<Language, List<String>> langLabDet) {
 370                         for (Language lang : langLabDet.keySet()){
 371                                 List<String> labDet = langLabDet.get(lang);
 372                                 if (labDet.size()>0){
 373                                         if (labDet.size()>1) {
 374                                                 tb.addRepresentation(Representation.NewInstance(labDet.get(1), labDet.get(0), labDet.get(0), lang));
 375                                         } else {
 376                                                 tb.addRepresentation(Representation.NewInstance(labDet.get(0), labDet.get(0), labDet.get(0), lang));
 377                                         }
 378                                 }
 379                         }
 380         }
 381
 382
 383         /**
 384          * Handles the "MediaObject" children of representations.
 385          * @param sddNamespace
 386          * @param ve
 387          * @param elRepresentation
 388          */
 389         private void makeRepresentationMediaObjects(Namespace sddNamespace,     VersionableEntity ve, Element elRepresentation) {
 390                 List <Element> listMediaObjects = elRepresentation.getChildren("MediaObject", sddNamespace);
 391                 for (Element elMediaObject : listMediaObjects) {
 392                         String ref = null;
 393                         //TODO
 394                         String role = null;
 395                         if (elMediaObject != null) {
 396                                 ref = elMediaObject.getAttributeValue("ref");
 397                                 role = elMediaObject.getAttributeValue("role");
 398                         }
 399                         if (StringUtils.isNotBlank(ref)) {
 400                                 if (ve instanceof TaxonDescription) {
 401                                         TaxonDescription td = (TaxonDescription) ve;
 402                                         if (td.getSources().size() > 0) {
 403                                                 this.associateImageWithCdmBase(ref, td.getSources().iterator().next().getCitation());
 404                                         } else {
 405                                                 Reference<?> descriptionSource = ReferenceFactory.newGeneric();
 406                                                 sources.add(descriptionSource);
 407                                                 //TODO type
 408                                                 td.addSource(OriginalSourceType.Unknown, null, null, descriptionSource, null);
 409                                                 this.associateImageWithCdmBase(ref,descriptionSource);
 410                                         }
 411                                 } else {
 412                                         this.associateImageWithCdmBase(ref,ve);
 413                                 }
 414                         }
 415                 }
 416         }
 417
 418         /**
 419          * Handles the "Links" element
 420          * @param parent
 421          * @param sddNamespace
 422          * @param ve
 423          */
 424         private void makeRepresentationForIdentifiableMediaEntity(Element parent,
 425                         Namespace sddNamespace, IdentifiableMediaEntity ime) {
 426                 Element elLinks = parent.getChild("Links",sddNamespace);
 427
 428                 if (elLinks != null) {
 429
 430                         //  <Link rel="Alternate" href="http://www.diversitycampus.net/people/hagedorn"/>
 431                         List<Element> listLinks = elLinks.getChildren("Link", sddNamespace);
 432                         Media link = Media.NewInstance();
 433                         MediaRepresentation mr = MediaRepresentation.NewInstance();
 434                         int k = 0;
 435                         //for each Link
 436                         for (Element elLink : listLinks){
 437
 438                                 try {
 439                                         //TODO
 440                                         String rel = elLink.getAttributeValue("rel");
 441                                         String href = elLink.getAttributeValue("href");
 442                                         URI uri = new URI(href);
 443                                         mr.addRepresentationPart(MediaRepresentationPart.NewInstance(uri, null));
 444                                         link.addRepresentation(mr);
 445                                         ime.addMedia(link);
 446
 447                                 } catch (Exception e) {
 448                                         //FIXME
 449                                         logger.warn("Import of Link " + k + " failed.");
 450                                 }
 451
 452                                 if ((++k % modCount) == 0){ logger.info("Links handled: " + k);}
 453
 454                         }
 455                 }
 456         }
 457
 458         /**
 459          * @param sddNamespace
 460          * @param ve
 461          * @param elRepresentation
 462          * @param langLabDet
 463          * @return
 464          */
 465         private void makeRepresentationForIdentifiableEntity(Namespace sddNamespace, IdentifiableEntity<?> ie,
 466                                         Element elRepresentation, Map<Language, List<String>> langLabDet) {
 467                 List<String> labDet = null;
 468
 469                 if (ie instanceof TaxonNameBase) {
 470                         if (langLabDet.keySet().contains(getTermService().getLanguageByIso("la"))) {
 471                                 labDet = langLabDet.get(getTermService().getLanguageByIso("la"));
 472                         } else if (langLabDet.keySet().contains(datasetLanguage)) {
 473                                 labDet = langLabDet.get(datasetLanguage);
 474                                 logger.info("TaxonName " + (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace) + " is not specified as a latin name.");
 475                         } else {
 476                                 labDet = langLabDet.get(langLabDet.keySet().iterator().next());
 477                                 logger.info("TaxonName " + (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace) + " is not specified as a latin name.");
 478                         }
 479                 } else {
 480                         labDet = langLabDet.get(langLabDet.keySet().iterator().next());
 481                 }
 482
 483                 //FIXME labDet is != null only for TaxonNameBase
 484                 ie.setTitleCache(labDet.get(0), true);
 485
 486                 if (labDet.size()>1) {
 487                         Annotation annotation = null;
 488                         if (labDet.get(1) != null) {
 489                                 if (labDet.get(2) != null) {
 490                                         annotation = Annotation.NewInstance(labDet.get(2) + " - " + labDet.get(1), datasetLanguage);
 491                                 } else {
 492                                         annotation = Annotation.NewInstance(labDet.get(1), datasetLanguage);
 493                                 }
 494                         }
 495                         ie.addAnnotation(annotation);
 496                 }
 497                 return;
 498         }
 499
 500         /**
 501          * @param elLabel
 502          * @return
 503          */
 504         private Language getLanguage(Element elLanguage) {
 505                 String lang = elLanguage.getAttributeValue("lang",xmlNamespace);
 506                 Language language = null;
 507                 if (StringUtils.isNotBlank(lang)) {
 508                         language = getTermService().getLanguageByIso(lang.substring(0, 2));
 509                 } else {
 510                         language = datasetLanguage;
 511                 }
 512                 return language;
 513         }
 514
 515
 516         // imports the representation (label, detail, lang) of a particular SDD element
 517         protected void importTechnicalMetadata(Element root, Namespace sddNamespace, SDDImportConfigurator sddConfig){
 518                 Element elTechnicalMetadata = root.getChild("TechnicalMetadata", sddNamespace);
 519                 String nameCreated = elTechnicalMetadata.getAttributeValue("created");
 520                 sourceReference = sddConfig.getSourceReference();
 521
 522                 if (nameCreated != null) {
 523                         if (!nameCreated.equals("")) {
 524                                 int year = Integer.parseInt(nameCreated.substring(0,4));
 525                                 int monthOfYear = Integer.parseInt(nameCreated.substring(5,7));
 526                                 int dayOfMonth = Integer.parseInt(nameCreated.substring(8,10));
 527                                 int hourOfDay = Integer.parseInt(nameCreated.substring(11,13));
 528                                 int minuteOfHour = Integer.parseInt(nameCreated.substring(14,16));
 529                                 int secondOfMinute = Integer.parseInt(nameCreated.substring(17,19));
 530                                 DateTime created = new DateTime(year,monthOfYear,dayOfMonth,hourOfDay,minuteOfHour,secondOfMinute,0);
 531                                 sourceReference.setCreated(created);
 532                                 sec.setCreated(created);
 533                         }
 534                 }
 535
 536                 // <Generator name="n/a, handcrafted instance document" version="n/a"/>
 537                 Element elGenerator = elTechnicalMetadata.getChild("Generator", sddNamespace);
 538                 generatorName = elGenerator.getAttributeValue("name");
 539                 generatorVersion = elGenerator.getAttributeValue("version");
 540
 541                 sec.addAnnotation(Annotation.NewDefaultLanguageInstance(generatorName + " - " + generatorVersion));
 542                 sourceReference.addAnnotation(Annotation.NewDefaultLanguageInstance(generatorName + " - " + generatorVersion));
 543
 544         }
 545
 546         // imports the complete dataset information
 547         protected void importDataset(Element elDataset, Namespace sddNamespace, SDDImportState state){                  // <Dataset xml:lang="en-us">
 548 //              SDDImportConfigurator sddConfig = state.getConfig();
 549
 550                 workingSet = WorkingSet.NewInstance();
 551                 importDatasetLanguage(elDataset,state);
 552                 importDatasetRepresentation(elDataset, sddNamespace);
 553                 importRevisionData(elDataset, sddNamespace);
 554                 importIPRStatements(elDataset, sddNamespace, state);
 555                 importTaxonNames(elDataset, sddNamespace, state);
 556
 557                 importDescriptiveConcepts(elDataset, sddNamespace, state);
 558                 importCharacters(elDataset, sddNamespace, state);
 559                 importCharacterTrees(elDataset, sddNamespace, state);
 560
 561                 MarkerType editorMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerEditor, "editor", "Editor", "edt");
 562                 MarkerType geographicAreaMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerSDDGeographicArea, "SDDGeographicArea", "SDDGeographicArea", "ga");
 563                 MarkerType descriptiveConceptMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerDescriptiveConcept, "DescriptiveConcept", "Descriptive Concept", "DC");
 564                 markerTypes.add(editorMarkerType);
 565                 markerTypes.add(geographicAreaMarkerType);
 566                 markerTypes.add(descriptiveConceptMarkerType);
 567
 568                 //saving of all imported data into the CDM db
 569                 saveVocabularies();
 570                 saveFeatures();
 571                 saveModifiers();
 572                 saveStates();
 573                 saveMarkerType();
 574                 saveAreas(geographicAreaMarkerType);
 575                 saveUnits();
 576                 saveStatisticalMeasure();
 577                 saveAnnotationType();
 578
 579                 importCodedDescriptions(elDataset, sddNamespace, state);
 580                 importAgents(elDataset, sddNamespace, state);
 581                 importPublications(elDataset, sddNamespace, state);
 582                 importMediaObjects(elDataset, sddNamespace, state);
 583                 importTaxonHierarchies(elDataset, sddNamespace, state);
 584                 importGeographicAreas(elDataset, sddNamespace, state);
 585                 importSpecimens(elDataset,sddNamespace, state);
 586
 587
 588                 if ((authors != null)||(editors != null)) {
 589                         Team team = Team.NewInstance();
 590                         if (authors != null) {
 591                                 for (Person author : authors.values()){
 592                                         team.addTeamMember(author);
 593                                 }
 594                         }
 595                         if (editors != null) {
 596                                 Marker marker = Marker.NewInstance();
 597                                 marker.setMarkerType(editorMarkerType);
 598                                 for (Person editor : editors.values()){
 599                                         Person edit = editor;
 600                                         edit.addMarker(marker);
 601                                         team.addTeamMember(edit);
 602                                 }
 603                         }
 604                         sec.setAuthorTeam(team);
 605                         sourceReference.setAuthorTeam(team);
 606                 }
 607
 608                 if (copyright != null) {
 609                         sourceReference.addRights(copyright);
 610                         sec.addRights(copyright);
 611                 }
 612
 613                 // Returns a CdmApplicationController created by the values of this configuration.
 614                 IDescriptionService descriptionService = getDescriptionService();
 615
 616                 for (TaxonDescription taxonDescription : taxonDescriptions.values()){
 617                         // Persists a Description
 618                         descriptionService.save(taxonDescription);
 619                 }
 620
 621                 for (String ref : taxonDescriptions.keySet()){
 622                         TaxonDescription td = taxonDescriptions.get(ref);
 623                         if (citations.containsKey(ref)) {
 624                                 Reference<?> publication = publications.get(citations.get(ref));
 625                                 if (locations.containsKey(ref)) {
 626                                         Annotation location = Annotation.NewInstance(locations.get(ref), datasetLanguage);
 627                                         AnnotationType annotationType = AnnotationType.NewInstance("", "location", "");
 628                                         annotationTypes.add(annotationType);
 629                                         location.setAnnotationType(annotationType);
 630                                         (publication).addAnnotation(location);
 631                                 }
 632                                 //TODO type
 633                                 td.addSource(OriginalSourceType.Unknown, null, null, publication, null);
 634                         }
 635                 }
 636                 logger.info("end makeTaxonDescriptions ...");
 637
 638                 if (descriptiveConcepts != null) {
 639                         for (Feature feature : descriptiveConcepts) {
 640                                 Marker marker = Marker.NewInstance();
 641                                 marker.setMarkerType(descriptiveConceptMarkerType);
 642                                 feature.addMarker(marker);
 643                         }
 644                 }
 645                 saveFeatures();
 646
 647                 for (Reference publication : publications.values()){
 648                         getReferenceService().save(publication);
 649                 }
 650
 651                 for (Reference source : sources){
 652                         getReferenceService().save(source);
 653                 }
 654
 655                 for (FeatureTree featureTree : featureTrees) {
 656                         getFeatureTreeService().save(featureTree);
 657                 }
 658                 getWorkingSetService().save(workingSet);
 659                 for (Classification classification : classifications) {
 660                         getClassificationService().save(classification);
 661                 }
 662                 for (DerivedUnit specimen : specimens.values()) {
 663                         getOccurrenceService().save(specimen);
 664                 }
 665                 logger.info("end of persistence ...");
 666
 667                 return;
 668         }
 669
 670         /**
 671          *
 672          */
 673         private void saveVocabularies() {
 674                 for (TermVocabulary vocabulary : vocabularies ){
 675                         getVocabularyService().save(vocabulary);
 676                 }
 677
 678         }
 679
 680         private void saveAnnotationType() {
 681                 for (AnnotationType annotationType: annotationTypes){
 682                         getTermService().save(annotationType);
 683                 }
 684         }
 685
 686         private void saveStatisticalMeasure() {
 687                 for (StatisticalMeasure sm : statisticalMeasures){
 688                         getTermService().save(sm);
 689                 }
 690         }
 691
 692         private void saveUnits() {
 693                 if (units != null) {
 694                         for (MeasurementUnit unit : units.values()){
 695                                 if (unit != null) {
 696                                         getTermService().save(unit);
 697                                 }
 698                         }
 699                 }
 700         }
 701
 702         private void saveAreas(MarkerType geographicAreaMarkerType) {
 703                 for (NamedArea area : namedAreas.values() ){
 704                         Marker marker = Marker.NewInstance();
 705                         marker.setMarkerType(geographicAreaMarkerType);
 706                         area.addMarker(marker);
 707                         getTermService().save(area);
 708                 }
 709         }
 710
 711         private void saveStates() {
 712                 for (State state : states.values() ){
 713                         getTermService().save(state);
 714                 }
 715         }
 716
 717         private void saveMarkerType() {
 718                 for (MarkerType markerType : markerTypes){
 719                         getTermService().save(markerType);
 720                 }
 721         }
 722
 723         private void saveModifiers() {
 724                 for (DefinedTerm modifier : modifiers.values() ){
 725                         getTermService().save(modifier);
 726                 }
 727         }
 728
 729         private void saveFeatures() {
 730                 for (Feature feature : features.values() ){
 731                         getTermService().save(feature);
 732                 }
 733         }
 734
 735         // imports the default language of the dataset
 736         protected void importDatasetLanguage(Element elDataset, SDDImportState state){
 737                 String nameLang = elDataset.getAttributeValue("lang",xmlNamespace);
 738
 739                 if (StringUtils.isNotBlank(nameLang)) {
 740                         String iso = nameLang.substring(0, 2);
 741                         datasetLanguage = getTermService().getLanguageByIso(iso);
 742                 } else {
 743                         datasetLanguage = Language.DEFAULT();
 744                 }
 745                 if (datasetLanguage == null) {
 746                         datasetLanguage = Language.DEFAULT();
 747                 }
 748         }
 749
 750         // imports the specimens
 751         protected void importSpecimens(Element elDataset, Namespace sddNamespace, SDDImportState cdmState) {
 752                 logger.info("start Specimens ...");
 753                 /*      <Specimens>
 754                         <Specimen id="sp1">
 755                                 <Representation>
 756                                 <Label>TJM45337</Label>
 757                                 </Representation>
 758                         </Specimen>
 759                 </Specimens>
 760                  */
 761                 Element elSpecimens = elDataset.getChild("Specimens",sddNamespace);
 762                 if (elSpecimens != null){
 763                         List<Element> listSpecimens = elSpecimens.getChildren("Specimen", sddNamespace);
 764                         int j = 0;
 765                         for (Element elSpecimen : listSpecimens) {
 766                                 String id = elSpecimen.getAttributeValue("id");
 767                                 DerivedUnit specimen = null;
 768                                 if (!id.equals("")) {
 769                                         specimen = DerivedUnit.NewPreservedSpecimenInstance();
 770                                         specimens.put(id,specimen);
 771                                         importRepresentation(elSpecimen, sddNamespace, specimen, id, cdmState);
 772                                 }
 773                         }
 774
 775                 }
 776         }
 777
 778         // imports the revision data associated with the Dataset (authors, modifications)
 779         protected void importRevisionData(Element elDataset, Namespace sddNamespace){
 780                 // <RevisionData>
 781                 logger.info("start RevisionData ...");
 782                 Element elRevisionData = elDataset.getChild("RevisionData",sddNamespace);
 783                 if (elRevisionData != null){
 784                         // <Creators>
 785                         Element elCreators = elRevisionData.getChild("Creators",sddNamespace);
 786
 787                         // <Agent role="aut" ref="a1"/>
 788                         List<Element> listAgents = elCreators.getChildren("Agent", sddNamespace);
 789
 790                         int j = 0;
 791                         //for each Agent
 792                         for (Element elAgent : listAgents){
 793
 794                                 String role = elAgent.getAttributeValue("role");
 795                                 String ref = elAgent.getAttributeValue("ref");
 796                                 if (role.equals("aut")) {
 797                                         if(!ref.equals("")) {
 798                                                 authors.put(ref, null);
 799                                         }
 800                                 }
 801                                 if (role.equals("edt")) {
 802                                         if(!ref.equals("")) {
 803                                                 editors.put(ref, null);
 804                                         }
 805                                 }
 806                                 if ((++j % modCount) == 0){ logger.info("Agents handled: " + j);}
 807
 808                         }
 809
 810                         // <DateModified>2006-04-08T00:00:00</DateModified>
 811                         String stringDateModified = (String)ImportHelper.getXmlInputValue(elRevisionData, "DateModified",sddNamespace);
 812
 813                         if (stringDateModified != null) {
 814                                 SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss");
 815                                 Date d = null;
 816                                 try {
 817                                         d = sdf.parse(stringDateModified);
 818                                 } catch(Exception e) {
 819                                         System.err.println("Exception :");
 820                                         e.printStackTrace();
 821                                 }
 822
 823                                 DateTime updated = null;
 824                                 if (d != null) {
 825                                         updated = new DateTime(d);
 826                                         sourceReference.setUpdated(updated);
 827                                         sec.setUpdated(updated);
 828                                 }
 829                         }
 830                 }
 831         }
 832
 833         // imports ipr statements associated with a dataset
 834         protected void importIPRStatements(Element elDataset, Namespace sddNamespace, SDDImportState state){
 835                 // <IPRStatements>
 836                 logger.info("start IPRStatements ...");
 837                 Element elIPRStatements = elDataset.getChild("IPRStatements",sddNamespace);
 838                 // <IPRStatement role="Copyright">
 839                 if (elIPRStatements != null) {
 840                         List<Element> listIPRStatements = elIPRStatements.getChildren("IPRStatement", sddNamespace);
 841                         int j = 0;
 842                         //for each IPRStatement
 843
 844                         for (Element elIPRStatement : listIPRStatements){
 845
 846                                 String role = elIPRStatement.getAttributeValue("role");
 847                                 // <Label xml:lang="en-au">(c) 2003-2006 Centre for Occasional Botany.</Label>
 848                                 Element elLabel = elIPRStatement.getChild("Label",sddNamespace);
 849                                 String lang = "";
 850                                 if (elLabel != null) {
 851                                         lang = elLabel.getAttributeValue("lang",xmlNamespace);
 852                                 }
 853                                 String label = (String)ImportHelper.getXmlInputValue(elIPRStatement, "Label",sddNamespace);
 854
 855                                 if (role.equals("Copyright")) {
 856                                         Language iprLanguage = null;
 857                                         if (lang != null) {
 858                                                 if (!lang.equals("")) {
 859                                                         iprLanguage = getTermService().getLanguageByIso(lang.substring(0, 2));
 860                                                 } else {
 861                                                         iprLanguage = datasetLanguage;
 862                                                 }
 863                                         }
 864                                         if (iprLanguage == null) {
 865                                                 iprLanguage = datasetLanguage;
 866                                         }
 867                                         copyright = Rights.NewInstance(label, iprLanguage);
 868                                 }
 869
 870                                 if (copyright != null) {
 871                                         sourceReference.addRights(copyright);
 872                                         sec.addRights(copyright);
 873                                 }
 874
 875                                 if ((++j % modCount) == 0){ logger.info("IPRStatements handled: " + j);}
 876
 877                         }
 878                 }
 879         }
 880
 881         // imports the taxon names
 882         protected void importTaxonNames(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
 883                 // <TaxonNames>
 884                 logger.info("start TaxonNames ...");
 885                 Element elTaxonNames = elDataset.getChild("TaxonNames",sddNamespace);
 886                 // <TaxonName id="t1" uri="urn:lsid:authority:namespace:my-own-id">
 887                 if (elTaxonNames != null) {
 888                         List<Element> listTaxonNames = elTaxonNames.getChildren("TaxonName", sddNamespace);
 889                         int j = 0;
 890                         //for each TaxonName
 891                         for (Element elTaxonName : listTaxonNames){
 892
 893                                 String id = elTaxonName.getAttributeValue("id");
 894                                 String uri = elTaxonName.getAttributeValue("uri");
 895
 896                                 NonViralName tnb = null;
 897                                 if (!id.equals("")) {
 898                                         tnb = NonViralName.NewInstance(null);
 899                                         IdentifiableSource source = null;
 900                                         if (isNotBlank(uri)) {
 901                                                 //TODO type
 902                                                 source = IdentifiableSource.NewInstance(OriginalSourceType.Unknown, id, "TaxonName", ReferenceFactory.newGeneric(), uri);
 903                                         } else {
 904                                                 source = IdentifiableSource.NewDataImportInstance(id, "TaxonName");
 905                                         }
 906                                         tnb.addSource(source);
 907                                         taxonNameBases.put(id,tnb);
 908                                 }
 909
 910                                 // <Representation>
 911                                 // <Label xml:lang="la">Viola hederacea Labill.</Label>
 912                                 importRepresentation(elTaxonName, sddNamespace, tnb, id, cdmState);
 913
 914                                 if ((++j % modCount) == 0){ logger.info("TaxonNames handled: " + j);}
 915
 916                         }
 917                 }
 918         }
 919
 920         // imports the characters (categorical, quantitative and text ; sequence characters not supported) which correspond to CDM Features
 921         protected void importCharacters(Element elDataset, Namespace sddNamespace, SDDImportState state){
 922                 // <Characters>
 923                 logger.info("start Characters ...");
 924                 Element elCharacters = elDataset.getChild("Characters", sddNamespace);
 925
 926                 // <CategoricalCharacter id="c1">
 927                 if (elCharacters != null) {
 928                         handleCategoricalData(sddNamespace, state, elCharacters);
 929                         handleQuantitativeData(sddNamespace, state, elCharacters);
 930                         handleTextCharacters(sddNamespace, state, elCharacters);
 931                 }
 932
 933                 /*for (Iterator<Feature> f = features.values().iterator() ; f.hasNext() ;){
 934                         featureSet.add(f.next()); //XIM Why this line ?
 935                 }*/
 936
 937                 return;
 938
 939         }
 940
 941         /**
 942          * @param sddNamespace
 943          * @param cdmState
 944          * @param elCharacters
 945          * @return
 946          */
 947         private void handleCategoricalData(Namespace sddNamespace, SDDImportState cdmState, Element elCharacters) {
 948                 List<Element> elCategoricalCharacters = elCharacters.getChildren("CategoricalCharacter", sddNamespace);
 949                 int j = 0;
 950                 for (Element elCategoricalCharacter : elCategoricalCharacters){
 951                         try {
 952
 953                                 String idCC = elCategoricalCharacter.getAttributeValue("id");
 954                                 Feature categoricalCharacter = Feature.NewInstance();
 955                                 categoricalCharacter.setKindOf(Feature.DESCRIPTION());
 956                                 importRepresentation(elCategoricalCharacter, sddNamespace, categoricalCharacter, idCC, cdmState);
 957                                 categoricalCharacter.setSupportsCategoricalData(true);
 958
 959                                 // <States>
 960                                 Element elStates = elCategoricalCharacter.getChild("States",sddNamespace);
 961
 962                                 // <StateDefinition id="s1">
 963                                 List<Element> elStateDefinitions = elStates.getChildren("StateDefinition",sddNamespace);
 964                                 TermVocabulary<State> termVocabularyState = TermVocabulary.NewInstance(TermType.State, null, null, null, null);
 965
 966                                 vocabularies.add(termVocabularyState);
 967
 968                                 int k = 0;
 969                                 //for each StateDefinition
 970                                 for (Element elStateDefinition : elStateDefinitions){
 971
 972                                         if ((++k % modCount) == 0){ logger.info("StateDefinitions handled: " + (k-1));}
 973
 974                                         String idS = elStateDefinition.getAttributeValue("id");
 975                                         State state = states.get(idS);
 976                                         if (state == null){
 977                                                 state = State.NewInstance();
 978                                         }else{
 979                                                 logger.debug("State duplicate found");
 980                                         }
 981                                         importRepresentation(elStateDefinition, sddNamespace, state, idS, cdmState);
 982
 983                                         termVocabularyState.addTerm(state);
 984                                         states.put(idS,state);
 985                                 }
 986                                 categoricalCharacter.addSupportedCategoricalEnumeration(termVocabularyState);
 987                                 features.put(idCC, categoricalCharacter);
 988
 989                         } catch (Exception e) {
 990                                 logger.warn("Import of CategoricalCharacter " + j + " failed.");
 991                                 cdmState.setUnsuccessfull();
 992                         }
 993
 994                         if ((++j % modCount) == 0){ logger.info("CategoricalCharacters handled: " + j);}
 995
 996                 }
 997                 return;
 998         }
 999
1000         /**
1001          * @param sddNamespace
1002          * @param sddConfig
1003          * @param elCharacters
1004          */
1005         private void handleQuantitativeData(Namespace sddNamespace,     SDDImportState cdmState, Element elCharacters) {
1006                 int j;
1007                 // <QuantitativeCharacter id="c2">
1008                 List<Element> elQuantitativeCharacters = elCharacters.getChildren("QuantitativeCharacter", sddNamespace);
1009                 j = 0;
1010                 //for each QuantitativeCharacter
1011                 for (Element elQuantitativeCharacter : elQuantitativeCharacters){
1012
1013                         try {
1014
1015                                 String idQC = elQuantitativeCharacter.getAttributeValue("id");
1016
1017                                 // <Representation>
1018                                 //  <Label>Leaf length</Label>
1019                                 // </Representation>
1020                                 Feature quantitativeCharacter = Feature.NewInstance();
1021                                 quantitativeCharacter.setKindOf(Feature.DESCRIPTION());
1022                                 importRepresentation(elQuantitativeCharacter, sddNamespace, quantitativeCharacter, idQC, cdmState);
1023
1024                                 quantitativeCharacter.setSupportsQuantitativeData(true);
1025
1026                                 // <MeasurementUnit>
1027                                 //  <Label role="Abbrev">m</Label>
1028                                 // </MeasurementUnit>
1029                                 Element elMeasurementUnit = elQuantitativeCharacter.getChild("MeasurementUnit",sddNamespace);
1030                                 String label = "";
1031                                 String role = "";
1032                                 if (elMeasurementUnit != null) {
1033                                         Element elLabel = elMeasurementUnit.getChild("Label",sddNamespace);
1034                                         role = elLabel.getAttributeValue("role");
1035                                         label = (String)ImportHelper.getXmlInputValue(elMeasurementUnit, "Label",sddNamespace);
1036                                 }
1037
1038                                 MeasurementUnit unit = null;
1039                                 if (!label.equals("")){
1040                                         if (role != null) {
1041                                                 if (role.equals("Abbrev")){
1042                                                         unit = MeasurementUnit.NewInstance(label,label,label);
1043                                                 }
1044                                         } else {
1045                                                 unit = MeasurementUnit.NewInstance(label,label,label);
1046                                         }
1047                                 }
1048
1049                                 if (unit != null) {
1050                                         units.put(idQC, unit);
1051                                 }
1052
1053                                 //<Default>
1054                                 //  <MeasurementUnitPrefix>milli</MeasurementUnitPrefix>
1055                                 //</Default>
1056                                 Element elDefault = elQuantitativeCharacter.getChild("Default",sddNamespace);
1057                                 if (elDefault != null) {
1058                                         String measurementUnitPrefix = (String)ImportHelper.getXmlInputValue(elDefault, "MeasurementUnitPrefix",sddNamespace);
1059                                         if (! measurementUnitPrefix.equals("")){
1060                                                 defaultUnitPrefixes.put(idQC, measurementUnitPrefix);
1061                                         }
1062                                 }
1063
1064                                 features.put(idQC, quantitativeCharacter);
1065
1066                         } catch (Exception e) {
1067                                 //FIXME
1068                                 logger.warn("Import of QuantitativeCharacter " + j + " failed.");
1069                                 cdmState.setUnsuccessfull();
1070                         }
1071
1072                         if ((++j % modCount) == 0){ logger.info("QuantitativeCharacters handled: " + j);}
1073
1074                 }
1075                 return;
1076         }
1077
1078         private void handleTextCharacters(Namespace sddNamespace, SDDImportState cdmState, Element elCharacters) {
1079                 int j;
1080                 // <TextCharacter id="c3">
1081                 List<Element> elTextCharacters = elCharacters.getChildren("TextCharacter", sddNamespace);
1082                 j = 0;
1083                 //for each TextCharacter
1084                 for (Element elTextCharacter : elTextCharacters){
1085
1086                         try {
1087
1088                                 String idTC = elTextCharacter.getAttributeValue("id");
1089
1090                                 // <Representation>
1091                                 //  <Label xml:lang="en">Leaf features not covered by other characters</Label>
1092                                 // </Representation>
1093                                 Feature textCharacter = Feature.NewInstance();
1094                                 textCharacter.setKindOf(Feature.DESCRIPTION());
1095                                 importRepresentation(elTextCharacter, sddNamespace, textCharacter, idTC, cdmState);
1096
1097                                 textCharacter.setSupportsTextData(true);
1098
1099                                 features.put(idTC, textCharacter);
1100
1101                         } catch (Exception e) {
1102                                 //FIXME
1103                                 logger.warn("Import of TextCharacter " + j + " failed.");
1104                                 cdmState.setUnsuccessfull();
1105                         }
1106
1107                         if ((++j % modCount) == 0){ logger.info("TextCharacters handled: " + j);}
1108
1109                 }
1110                 return;
1111         }
1112
1113         // imports the descriptions of taxa
1114         protected void importCodedDescriptions(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1115
1116                 // <CodedDescriptions>
1117                 logger.info("start CodedDescriptions ...");
1118                 Element elCodedDescriptions = elDataset.getChild("CodedDescriptions",sddNamespace);
1119
1120                 // <CodedDescription id="D101">
1121                 if (elCodedDescriptions != null) {
1122                         List<Element> listCodedDescriptions = elCodedDescriptions.getChildren("CodedDescription", sddNamespace);
1123                         int j = 0;
1124                         //for each CodedDescription
1125                         for (Element elCodedDescription : listCodedDescriptions){
1126                                 handleCodedDescription(sddNamespace, cdmState, elCodedDescription, j);
1127                                 if ((++j % modCount) == 0){ logger.info("CodedDescriptions handled: " + j);}
1128                         }
1129                 }
1130                 return;
1131         }
1132
1133         /**
1134          * @param sddNamespace
1135          * @param sddConfig
1136          * @param j
1137          * @param elCodedDescription
1138          * @return
1139          */
1140         private void handleCodedDescription(Namespace sddNamespace, SDDImportState cdmState, Element elCodedDescription, int j) {
1141                 try {
1142
1143                         String idCD = elCodedDescription.getAttributeValue("id");
1144
1145                         // <Representation>
1146                         //  <Label>&lt;i&gt;Viola hederacea&lt;/i&gt; Labill. as revised by R. Morris April 8, 2006</Label>
1147                         // </Representation>
1148                         TaxonDescription taxonDescription = TaxonDescription.NewInstance();
1149                         if (!generatorName.isEmpty()){
1150                                 Annotation annotation = Annotation.NewInstance(generatorName, AnnotationType.TECHNICAL(),Language.DEFAULT());
1151                                 taxonDescription.addAnnotation(annotation);
1152                         }
1153                         importRepresentation(elCodedDescription, sddNamespace, taxonDescription, idCD, cdmState);
1154
1155                         // <Scope>
1156                         //  <TaxonName ref="t1"/>
1157                         //  <Citation ref="p1" location="p. 30"/>
1158                         // </Scope>
1159                         Element elScope = elCodedDescription.getChild("Scope", sddNamespace);
1160                         Taxon taxon;
1161                         if (elScope != null) {
1162                                 taxon = handleCDScope(sddNamespace, cdmState, idCD, elScope);
1163                         } else {//in case no taxon is linked to the description, a new one is created
1164                                 taxon = handleCDNoScope(sddNamespace, cdmState, elCodedDescription);
1165                         }
1166
1167                         // <SummaryData>
1168                         Element elSummaryData = elCodedDescription.getChild("SummaryData",sddNamespace);
1169                         if (elSummaryData != null) {
1170                                 handleSummaryCategoricalData(sddNamespace, taxonDescription, elSummaryData);
1171                                 handleSummaryQuantitativeData(sddNamespace, taxonDescription, elSummaryData);
1172                                 handleSummaryTextData(sddNamespace, taxonDescription, elSummaryData);
1173                         }
1174
1175                         if (taxon != null) {
1176                                 taxon.addDescription(taxonDescription);
1177                         }
1178 //
1179                         workingSet.addDescription(taxonDescription);
1180
1181 //OLD                   taxonDescription.setDescriptiveSystem(featureSet);
1182
1183                         taxonDescriptions.put(idCD, taxonDescription);//FIXME
1184
1185                 } catch (Exception e) {
1186                         //FIXME
1187                         logger.warn("Import of CodedDescription " + j + " failed.", e);
1188                         cdmState.setUnsuccessfull();
1189                 }
1190                 return;
1191         }
1192
1193         /**
1194          * @param sddNamespace
1195          * @param sddConfig
1196          * @param elCodedDescription
1197          * @param taxon
1198          * @return
1199          */
1200         private Taxon handleCDNoScope(Namespace sddNamespace,
1201                         SDDImportState cdmState, Element elCodedDescription     ) {
1202                 Taxon taxon = null;
1203                 NonViralName<?> nonViralName = NonViralName.NewInstance(null);
1204                 String id = new String("" + taxonNamesCount);
1205                 IdentifiableSource source = IdentifiableSource.NewDataImportInstance( id, "TaxonName");
1206                 importRepresentation(elCodedDescription, sddNamespace, nonViralName, id, cdmState);
1207
1208                 if(cdmState.getConfig().isDoMatchTaxa()){
1209                         taxon = getTaxonService().findBestMatchingTaxon(nonViralName.getTitleCache());
1210                 }
1211
1212                 if(taxon != null){
1213                         nonViralName = HibernateProxyHelper.deproxy(taxon.getName(), NonViralName.class);
1214 //                                                      taxonNameBases.put(id ,tnb);
1215 //                                                      taxonNamesCount++;
1216                         logger.info("using existing Taxon " + taxon.getTitleCache());
1217                 } else {
1218                         nonViralName.addSource(source);
1219                         taxonNameBases.put(id ,nonViralName);
1220                         taxonNamesCount++;
1221                         logger.info("creating new Taxon from TaxonName " + nonViralName.getTitleCache());
1222                         taxon = Taxon.NewInstance(nonViralName, sec);
1223                 }
1224                 return taxon;
1225         }
1226
1227         /**
1228          * @param sddNamespace
1229          * @param sddConfig
1230          * @param idCD
1231          * @param elScope
1232          * @param taxon
1233          * @return
1234          */
1235         private Taxon handleCDScope(Namespace sddNamespace, SDDImportState cdmState,
1236                         String idCD, Element elScope) {
1237                 Taxon taxon = null;
1238                 Element elTaxonName = elScope.getChild("TaxonName", sddNamespace);
1239                 String ref = elTaxonName.getAttributeValue("ref");
1240                 NonViralName nonViralName = taxonNameBases.get(ref);
1241
1242                 if(cdmState.getConfig().isDoMatchTaxa()){
1243                         taxon = getTaxonService().findBestMatchingTaxon(nonViralName.getTitleCache());
1244                 }
1245
1246                 if(taxon != null){
1247                         logger.info("using existing Taxon" + taxon.getTitleCache());
1248                         if(!nonViralName.getUuid().equals(taxon.getName().getUuid())){
1249                                 logger.warn("TaxonNameBase entity of existing taxon does not match Name in list -> replacing Name in list");
1250                                 nonViralName = HibernateProxyHelper.deproxy(taxon.getName(), NonViralName.class);
1251                         }
1252                 } else {
1253                         logger.info("creating new Taxon from TaxonName '" + nonViralName.getTitleCache()+"'");
1254                         taxon = Taxon.NewInstance(nonViralName, sec);
1255                 }
1256
1257                 //citation
1258                 Element elCitation = elScope.getChild("Citation",sddNamespace);
1259                 if (elCitation != null) {
1260                         String refCitation = elCitation.getAttributeValue("ref");
1261                         if (! refCitation.equals("")){
1262                                 citations.put(idCD, refCitation);
1263                         }
1264                         String location = elCitation.getAttributeValue("location");
1265                         if (! location.equals("")){
1266                                 locations.put(idCD, location);
1267                         }
1268                 }
1269                 return taxon;
1270         }
1271
1272         /**
1273          * @param sddNamespace
1274          * @param taxonDescription
1275          * @param elSummaryData
1276          */
1277         private void handleSummaryTextData(Namespace sddNamespace,
1278                         TaxonDescription taxonDescription, Element elSummaryData) {
1279                 String ref;
1280                 int k;
1281                 // <TextChar ref="c3">
1282                 List<Element> elTextChars = elSummaryData.getChildren("TextChar", sddNamespace);
1283                 k = 0;
1284                 //for each TextChar
1285                 for (Element elTextChar : elTextChars){
1286                         if ((++k % modCount) == 0){ logger.info("TextChar handled: " + (k-1));}
1287                         ref = elTextChar.getAttributeValue("ref");
1288                         Feature feature = features.get(ref);
1289                         TextData textData = TextData.NewInstance();
1290                         textData.setFeature(feature);
1291
1292                         // <Content>Free form text</Content>
1293                         String content = (String)ImportHelper.getXmlInputValue(elTextChar, "Content",sddNamespace);
1294                         textData.putText(datasetLanguage, content);
1295                         taxonDescription.addElement(textData);
1296                 }
1297         }
1298
1299         /**
1300          * @param sddNamespace
1301          * @param taxonDescription
1302          * @param elSummaryData
1303          */
1304         private void handleSummaryQuantitativeData(Namespace sddNamespace,
1305                         TaxonDescription taxonDescription, Element elSummaryData) {
1306                 String ref;
1307                 int k;
1308                 // <Quantitative ref="c2">
1309                 List<Element> elQuantitatives = elSummaryData.getChildren("Quantitative", sddNamespace);
1310                 k = 0;
1311                 //for each Quantitative
1312                 for (Element elQuantitative : elQuantitatives){
1313                         if ((++k % modCount) == 0){ logger.warn("Quantitative handled: " + (k-1));}
1314                         ref = elQuantitative.getAttributeValue("ref");
1315                         Feature feature = features.get(ref);
1316                         QuantitativeData quantitativeData = QuantitativeData.NewInstance();
1317                         quantitativeData.setFeature(feature);
1318
1319                         MeasurementUnit unit = units.get(ref);
1320                         String prefix = defaultUnitPrefixes.get(ref);
1321                         if (unit != null) {
1322                                 String u = unit.getLabel();
1323                                 if (prefix != null) {
1324                                         u = prefix + u;
1325                                 }
1326                                 unit.setLabel(u);
1327                                 quantitativeData.setUnit(unit);
1328                         }
1329
1330                         // <Measure type="Min" value="2.3"/>
1331                         List<Element> elMeasures = elQuantitative.getChildren("Measure", sddNamespace);
1332                         int l = 0;
1333
1334                         //for each State
1335                         for (Element elMeasure : elMeasures){
1336                                 if ((++l % modCount) == 0){ logger.info("States handled: " + (l-1));}
1337                                 String type = elMeasure.getAttributeValue("type");
1338                                 String value = elMeasure.getAttributeValue("value");
1339                                 if (value.contains(",")) {
1340                                         value = value.replace(',', '.');
1341                                 }
1342                                 Float v = Float.parseFloat(value);
1343                                 //Float v = new Float(0);
1344                                 StatisticalMeasure t = null;
1345                                 if (type.equals("Min")) {
1346                                         t = StatisticalMeasure.MIN();
1347                                 } else if (type.equals("Mean")) {
1348                                         t = StatisticalMeasure.AVERAGE();
1349                                 } else if (type.equals("Max")) {
1350                                         t = StatisticalMeasure.MAX();
1351                                 } else if (type.equals("SD")) {
1352                                         t = StatisticalMeasure.STANDARD_DEVIATION();
1353                                 } else if (type.equals("N")) {
1354                                         t = StatisticalMeasure.SAMPLE_SIZE();
1355                                 } else if (type.equals("UMethLower")) {
1356                                         t = StatisticalMeasure.TYPICAL_LOWER_BOUNDARY();
1357                                 } else if (type.equals("UMethUpper")) {
1358                                         t = StatisticalMeasure.TYPICAL_UPPER_BOUNDARY();
1359                                 } else if (type.equals("Var")) {
1360                                         t = StatisticalMeasure.VARIANCE();
1361                                 } else {
1362                                         t = StatisticalMeasure.NewInstance(type,type,type);
1363                                         statisticalMeasures.add(t);
1364                                 }
1365
1366                                 StatisticalMeasurementValue statisticalValue = StatisticalMeasurementValue.NewInstance();
1367                                 statisticalValue.setValue(v);
1368                                 statisticalValue.setType(t);
1369                                 quantitativeData.addStatisticalValue(statisticalValue);
1370                                 featureData.add(statisticalValue);
1371                         }
1372                         taxonDescription.addElement(quantitativeData);
1373                 }
1374         }
1375
1376         /**
1377          * @param sddNamespace
1378          * @param taxonDescription
1379          * @param elSummaryData
1380          */
1381         private void handleSummaryCategoricalData(Namespace sddNamespace,
1382                         TaxonDescription taxonDescription, Element elSummaryData) {
1383                 String ref;
1384                 // <Categorical ref="c4">
1385                 List<Element> elCategoricals = elSummaryData.getChildren("Categorical", sddNamespace);
1386                 int k = 0;
1387                 //for each Categorical
1388                 for (Element elCategorical : elCategoricals){
1389                         if ((++k % modCount) == 0){ logger.warn("Categorical handled: " + (k-1));}
1390                         ref = elCategorical.getAttributeValue("ref");
1391                         Feature feature = features.get(ref);
1392                         CategoricalData categoricalData = CategoricalData.NewInstance();
1393                         categoricalData.setFeature(feature);
1394
1395                         // <State ref="s3"/>
1396                         List<Element> elStates = elCategorical.getChildren("State", sddNamespace);
1397                         int l = 0;
1398
1399                         //for each State
1400                         for (Element elState : elStates){
1401                                 if ((++l % modCount) == 0){ logger.info("States handled: " + (l-1));}
1402                                 ref = elState.getAttributeValue("ref");
1403                                 State state = states.get(ref);
1404                                 if (state != null) {
1405                                         StateData stateData = StateData.NewInstance();
1406                                         stateData.setState(state);
1407                                         List<Element> elModifiers = elState.getChildren("Modifier", sddNamespace);
1408                                         for (Element elModifier : elModifiers){
1409                                                 ref = elModifier.getAttributeValue("ref");
1410                                                 DefinedTerm modifier = modifiers.get(ref);
1411                                                 if (modifier != null) {
1412                                                         stateData.addModifier(modifier);
1413                                                 }
1414                                         }
1415                                         categoricalData.addStateData(stateData);
1416                                 }
1417                                 taxonDescription.addElement(categoricalData);
1418                         }
1419                 }
1420         }
1421
1422         // imports the persons associated with the dataset creation, modification, related publications
1423         protected void importAgents(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1424                 // <Agents>
1425                 logger.info("start Agents ...");
1426                 Element elAgents = elDataset.getChild("Agents",sddNamespace);
1427                 if (elAgents != null) {
1428                         // <Agent id="a1">
1429                         List <Element> listAgents = elAgents.getChildren("Agent", sddNamespace);
1430                         int j = 0;
1431                         //for each Agent
1432                         for (Element elAgent : listAgents){
1433
1434                                 try {
1435
1436                                         String idA = elAgent.getAttributeValue("id");
1437
1438                                         //  <Representation>
1439                                         //   <Label>Kevin Thiele</Label>
1440                                         //   <Detail role="Description">Ali Baba is also known as r.a.m.</Detail>
1441                                         //  </Representation>
1442                                         Person person = Person.NewInstance();
1443                                         importRepresentation(elAgent, sddNamespace, person, idA, cdmState);
1444                                         person.addSource(IdentifiableSource.NewDataImportInstance(idA, "Agent"));
1445
1446                                         /*XIM <Links>
1447                                         Element elLinks = elAgent.getChild("Links",sddNamespace);
1448
1449                                         if (elLinks != null) {
1450
1451                                                 //  <Link rel="Alternate" href="http://www.diversitycampus.net/people/hagedorn"/>
1452                                                 List<Element> listLinks = elLinks.getChildren("Link", sddNamespace);
1453                                                 int k = 0;
1454                                                 //for each Link
1455                                                 for (Element elLink : listLinks){
1456
1457                                                         try {
1458
1459                                                                 String rel = elLink.getAttributeValue("rel");
1460                                                                 String href = elLink.getAttributeValue("href");
1461
1462                                                                 Media link = Media.NewInstance();
1463                                                                 MediaRepresentation mr = MediaRepresentation.NewInstance();
1464                                                                 mr.addRepresentationPart(MediaRepresentationPart.NewInstance(href, null));
1465                                                                 link.addRepresentation(mr);
1466                                                                 person.addMedia(link);
1467
1468                                                         } catch (Exception e) {
1469                                                                 //FIXME
1470                                                                 logger.warn("Import of Link " + k + " failed.");
1471                                                                 success = false;
1472                                                         }
1473
1474                                                         if ((++k % modCount) == 0){ logger.info("Links handled: " + k);}
1475
1476                                                 }
1477                                         }
1478                                         */
1479                                         if (authors.containsKey(idA)) {
1480                                                 authors.put(idA,person);
1481                                         }
1482
1483                                         if (editors.containsKey(idA)) {
1484                                                 editors.put(idA, person);
1485                                         }
1486
1487                                 } catch (Exception e) {
1488                                         //FIXME
1489                                         logger.warn("Import of Agent " + j + " failed.");
1490                                         cdmState.setUnsuccessfull();
1491                                 }
1492
1493                                 if ((++j % modCount) == 0){ logger.info("Agents handled: " + j);}
1494
1495                         }
1496                 }
1497         }
1498
1499         // imports publications related with the data set
1500         protected void importPublications(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1501                 /* <Publications>
1502                           <Publication id="p112">
1503                             <Representation>
1504                               <Label>Gee, X. & Haa, Y. (2003). How to be happy in five minutes. Instant Gratifications, Palm Beach.</Label>
1505                             </Representation>
1506                             <Links>
1507                             <Link rel="BasedOn" href="doi:10.1992/32311"/>
1508                             <Link rel="Alternate" href="http://some.service.net/providing/bibliographic.data"/>
1509                             </Links>
1510                         </Publications>
1511 */
1512                 logger.info("start Publications ...");
1513                 Element elPublications = elDataset.getChild("Publications",sddNamespace);
1514
1515                 if (elPublications != null) {
1516                         List<Element> listPublications = elPublications.getChildren("Publication", sddNamespace);
1517                         int j = 0;
1518                         for (Element elPublication : listPublications){
1519
1520                                 try {
1521
1522                                         String idP = elPublication.getAttributeValue("id");
1523                                         Reference publication = ReferenceFactory.newArticle();
1524                                         importRepresentation(elPublication, sddNamespace, publication, idP, cdmState);
1525
1526                                         publications.put(idP,publication);
1527
1528                                 } catch (Exception e) {
1529                                         logger.warn("Import of Publication " + j + " failed.");
1530                                         cdmState.setUnsuccessfull();
1531                                 }
1532
1533                                 if ((++j % modCount) == 0){ logger.info("Publications handled: " + j);}
1534
1535                         }
1536                 }
1537         }
1538
1539         // imports media objects such as images //FIXME check mediaobj
1540         protected void importMediaObjects(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1541                 // <MediaObjects>
1542                 logger.info("start MediaObjects ...");
1543                 Element elMediaObjects = elDataset.getChild("MediaObjects",sddNamespace);
1544
1545                 if (elMediaObjects != null) {
1546                         // <MediaObject id="m1">
1547                         List<Element> listMediaObjects = elMediaObjects.getChildren("MediaObject", sddNamespace);
1548                         int j = 0;
1549                         for (Element elMO : listMediaObjects){
1550
1551                                 String id = "";
1552
1553                                 try {
1554                                         String idMO = elMO.getAttributeValue("id");
1555                                         id = idMO;
1556
1557                                         //  <Representation>
1558                                         //   <Label>Image description, e.g. to be used for alt-attribute in html.</Label>
1559                                         //  </Representation>
1560                                         Media media = Media.NewInstance();
1561                                         importRepresentation(elMO, sddNamespace, media, idMO, cdmState);
1562
1563                                         // <Type>Image</Type>
1564                                         // <Source href="http://test.edu/test.jpg"/>
1565                                         String type = (String)ImportHelper.getXmlInputValue(elMO,"Type",sddNamespace);
1566
1567                                         if ((type != null) && (type.equals("Image"))) {
1568                                                 Element elSource = elMO.getChild("Source",sddNamespace);
1569                                                 String href = elSource.getAttributeValue("href");
1570
1571                                                 ImageInfo imageMetaData = null;
1572                                                 ImageFile image = null;
1573                                                 if (href.substring(0,7).equals("http://")) {
1574                                                         try{
1575                                                                 URL url = new URL(href);
1576
1577                                                                 imageMetaData = ImageInfo.NewInstance(url.toURI(), 0);
1578                                                                 image = ImageFile.NewInstance(url.toURI(), null, imageMetaData);
1579                                                         } catch (MalformedURLException e) {
1580                                                                 logger.error("Malformed URL", e);
1581                                                         }
1582                                                 } else {
1583                                                         String sns = cdmState.getConfig().getSourceNameString();
1584                                                         File f = new File(sns);
1585                                                         File parent = f.getParentFile();
1586                                                         String fi = parent.toString() + File.separator + href;
1587                                                         File file = new File(fi);
1588                                                         imageMetaData = ImageInfo.NewInstance(new URI(fi), 0); //file
1589                                                         image = ImageFile.NewInstance(file.toURI(), null, imageMetaData);
1590                                                 }
1591                                                 MediaRepresentation representation = MediaRepresentation.NewInstance(imageMetaData.getMimeType(), null);
1592                                                 representation.addRepresentationPart(image);
1593
1594                                                 media.addRepresentation(representation);
1595
1596                                                 ArrayList<CdmBase> lcb = (ArrayList<CdmBase>) mediaObject_ListCdmBase.get(idMO);
1597                                                 if (lcb != null) {
1598                                                         for (int k = 0; k < lcb.size(); k++) {
1599                                                                 if (lcb.get(k) instanceof DefinedTermBase) {
1600                                                                         DefinedTermBase dtb = (DefinedTermBase) lcb.get(k);
1601                                                                         // if (lcb.get(0) instanceof DefinedTermBase) {
1602                                                                         // DefinedTermBase dtb = (DefinedTermBase) lcb.get(0);
1603                                                                         //                                                                      if (dtb!=null) {
1604                                                                         //                                                                              if (k == 0) {
1605                                                                         dtb.addMedia(media);
1606                                                                         //System.out.println(dtb.getLabel());
1607                                                                         //                                                                              } else {
1608                                                                         //                                                                                      Media me = (Media) media.clone();
1609                                                                         //                                                                                      dtb.addMedia(me);
1610                                                                         //                                                                              }
1611                                                                         //                                                                      }
1612                                                                 } else if (lcb.get(k) instanceof Reference) {
1613                                                                         Reference rb = (Reference) lcb.get(k);
1614                                                                         //} else if (lcb.get(0) instanceof Reference) {
1615                                                                         //Reference rb = (Reference) lcb.get(0);
1616                                                                         // rb.setTitleCache(label);
1617                                                                         //                                                                      if (rb!=null) {
1618                                                                         //                                                                              if (k == 0) {
1619                                                                         rb.addMedia(media);
1620                                                                         //System.out.println(rb.getTitle());
1621                                                                         //                                                                              } else {
1622                                                                         //                                                                                      Media me = (Media) media.clone();
1623                                                                         //                                                                                      rb.addMedia(me);
1624                                                                         //                                                                              }
1625                                                                         //                                                                      }
1626                                                                 }/* else if (lcb.get(k) instanceof TaxonNameBase){
1627                                                                         TaxonNameBase tb = (TaxonNameBase) lcb.get(k);
1628                                                                         tb.addMedia(media);
1629                                                         }*/
1630                                                         }
1631                                                 }
1632                                         }
1633
1634                                 } catch (Exception e) {
1635                                         //FIXME
1636                                         logger.warn("Could not attach MediaObject " + j + "(SDD: " + id + ") to several objects.");
1637                                         cdmState.setUnsuccessfull();
1638                                 }
1639
1640                                 if ((++j % modCount) == 0){ logger.info("MediaObjects handled: " + j);
1641
1642                                 }
1643                         }
1644                 }
1645         }
1646
1647         // imports the <DescriptiveConcepts> block ; DescriptiveConcepts are used as nodes in CharacterTrees and Characters as leaves
1648         // but since Modifiers can be linked to DescriptiveConcepts they are stored as features with a particular Marker
1649         protected void importDescriptiveConcepts(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1650                 /* <DescriptiveConcepts>
1651                       <DescriptiveConcept id="dc0">
1652                                 <Representation>
1653                                   <Label>Fixed set of modifiers supported in Lucid3</Label>
1654                                 </Representation>
1655                                 <Modifiers>
1656                                   <Modifier id="mod1">
1657                                     <Representation>
1658                                       <Label>rarely</Label>
1659                                     </Representation>
1660                                     <ModifierClass>Frequency</ModifierClass>
1661                                     <ProportionRange lowerestimate="0.0" upperestimate="0.25"/>
1662                                   </Modifier>
1663                           </Modifiers>
1664                         </DescriptiveConcept>
1665                  </DescriptiveConcepts>
1666                  */
1667                 logger.info("start DescriptiveConcepts ...");
1668                 Element elDescriptiveConcepts = elDataset.getChild("DescriptiveConcepts",sddNamespace);
1669                 if (elDescriptiveConcepts != null) {
1670                         List<Element> listDescriptiveConcepts = elDescriptiveConcepts.getChildren("DescriptiveConcept", sddNamespace);
1671                         int j = 0;
1672
1673                         for (Element elDescriptiveConcept : listDescriptiveConcepts){
1674                                 try {
1675                                 String id = elDescriptiveConcept.getAttributeValue("id");
1676                                         Feature feature = Feature.NewInstance();
1677                                         feature.setKindOf(Feature.DESCRIPTION());
1678                                         if (!id.equals("")) {
1679                                         //       <Representation>
1680                                         //       <Label>Body</Label>
1681                                         importRepresentation(elDescriptiveConcept, sddNamespace, feature, id, cdmState);
1682                                                 features.put(id, feature);
1683                                                 getTermService().save(feature);//XIM
1684                                                 descriptiveConcepts.add(feature);
1685                                                 // imports the modifiers
1686                                                 Element elModifiers = elDescriptiveConcept.getChild("Modifiers", sddNamespace);
1687                                         if (elModifiers !=null){
1688                                                 List<Element> listModifiers = elModifiers.getChildren("Modifier", sddNamespace);
1689                                                         TermVocabulary<DefinedTerm> termVocabularyState = TermVocabulary.NewInstance(TermType.Modifier, null, null, null, null);
1690                                                 for (Element elModifier : listModifiers) {
1691                                                         DefinedTerm modif = DefinedTerm.NewModifierInstance(null, null, null);
1692                                                         String idmod = elModifier.getAttributeValue("id");
1693                                                         importRepresentation(elModifier, sddNamespace, modif, idmod, cdmState);
1694                                                         termVocabularyState.addTerm(modif);
1695                                                         //termVocabularyStates.add(termVocabularyState);
1696                                                         getVocabularyService().save(termVocabularyState);//XIM
1697                                                         modifiers.put(idmod, modif);
1698                                                 }
1699                                                 feature.addRecommendedModifierEnumeration(termVocabularyState);
1700                                         }
1701
1702                                         }
1703                                 }
1704                                 catch (Exception e) {
1705                                         logger.warn("Import of DescriptiveConcept " + j + " failed: " + e.getMessage());
1706                                 }
1707                                 if ((++j % modCount) == 0){ logger.info("DescriptiveConcepts handled: " + j);}
1708
1709                         }
1710                 }
1711         }
1712
1713         // imports the <CharacterTrees> block
1714         protected void importCharacterTrees(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1715                 // <CharacterTrees>
1716                 logger.info("start CharacterTrees ...");
1717                 Element elCharacterTrees = elDataset.getChild("CharacterTrees",sddNamespace);
1718
1719                 if (elCharacterTrees != null) {
1720                         List<Element> listCharacterTrees = elCharacterTrees.getChildren("CharacterTree", sddNamespace);
1721                         int j = 0;
1722                         for (Element elCharacterTree : listCharacterTrees){
1723                                 try {
1724                                         Element elRepresentation = elCharacterTree.getChild("Representation",sddNamespace);
1725                                         String label = (String)ImportHelper.getXmlInputValue(elRepresentation,"Label",sddNamespace);
1726                                         //Element elDesignedFor = elCharacterTree.getChild("DesignedFor",sddNamespace);//TODO ?
1727
1728                                         FeatureTree featureTree =  FeatureTree.NewInstance();
1729                                         importRepresentation(elCharacterTree, sddNamespace, featureTree, "", cdmState);
1730                                         FeatureNode root = featureTree.getRoot();
1731                                         List<Element> listeOfNodes = elCharacterTree.getChildren("Nodes", sddNamespace);
1732
1733                                         //Nodes of CharacterTrees in SDD always refer to DescriptiveConcepts
1734                                         for (Element elNodes : listeOfNodes) {
1735                                                 handleCharacterNodes(sddNamespace, root, elNodes);
1736                                         }
1737                                         featureTrees.add(featureTree);
1738                                         if (workingSet.getDescriptiveSystem() != null){
1739                                                 //TODO how to handle multiple
1740                                                 logger.warn("Multiple feature trees not yet supported");
1741                                         }else{
1742                                                 workingSet.setDescriptiveSystem(featureTree);
1743                                         }
1744                                 }
1745
1746                                 catch (Exception e) {
1747                                         logger.warn("Import of Character tree " + j + " failed.");
1748                                         cdmState.setUnsuccessfull();
1749                                 }
1750                                 if ((++j % modCount) == 0){ logger.info("CharacterTrees handled: " + j);}
1751
1752                         }
1753
1754                 }
1755         }
1756
1757         /**
1758          * @param sddNamespace
1759          * @param root
1760          * @param elNodes
1761          */
1762         private void handleCharacterNodes(Namespace sddNamespace, FeatureNode root, Element elNodes) {
1763                 List<Element> listNodes = elNodes.getChildren("Node", sddNamespace);
1764                 if (listNodes != null) {
1765                         for (Element elNode : listNodes){
1766                                 String idN = elNode.getAttributeValue("id");
1767                                 FeatureNode fn = null;
1768                                 Feature dc = null;
1769                                 if (idN!=null) {
1770                                         // DescriptiveConcepts are used as nodes in CharacterTrees
1771                                         Element elDescriptiveConcept = elNode.getChild("DescriptiveConcept", sddNamespace);
1772                                         if (elDescriptiveConcept != null){
1773                                                 String refDC = elDescriptiveConcept.getAttributeValue("ref");
1774                                                 dc = features.get(refDC);
1775                                                 fn = FeatureNode.NewInstance(dc);
1776                                         }
1777                                         if (fn==null){
1778                                                 fn = FeatureNode.NewInstance();
1779                                         }
1780                                         Element elParent = elNode.getChild("Parent", sddNamespace);
1781                                         // in SDD links between Nodes are referenced by the <Parent> tag
1782                                         if (elParent!=null){
1783                                                 String refP = elParent.getAttributeValue("ref");
1784                                                 if (refP!=null) {
1785                                                         FeatureNode parent = featureNodes.get(refP);
1786                                                         if (parent==null){
1787                                                                 root.addChild(fn); // if no parent found or the reference is broken, add the node to the root of the tree
1788                                                         }
1789                                                         else {
1790                                                                 parent.addChild(fn);
1791                                                         }
1792                                                 }
1793                                         }
1794                                         else {
1795                                                 root.addChild(fn); // if no parent found or the reference is broken, add the node to the root of the tree
1796                                         }
1797                                 }
1798                                 featureNodes.put(idN, fn);
1799                         }
1800                 }
1801
1802                 // Leaves of CharacterTrees in SDD are always CharNodes (referring to Characters)
1803                 List<Element> listCharNodes = elNodes.getChildren("CharNode", sddNamespace);
1804                 if (listCharNodes != null) {
1805                         for (Element elCharNode : listCharNodes){
1806                                 Element elParent = elCharNode.getChild("Parent", sddNamespace);
1807                                 Element elCharacter = elCharNode.getChild("Character", sddNamespace);
1808                                 Element elDependencyRules = elCharNode.getChild("DependencyRules", sddNamespace);
1809                                 FeatureNode fn = FeatureNode.NewInstance();
1810
1811                                 if (elDependencyRules!=null){
1812                                         Element elInapplicableIf = elCharNode.getChild("InapplicableIf", sddNamespace);
1813                                         if (elInapplicableIf!=null){
1814                                                 List<Element> listStates = elInapplicableIf.getChildren("State", sddNamespace);
1815                                                 for (Element stateElement : listStates) {
1816                                                         String refState = stateElement.getAttributeValue("ref");
1817                                                         if ((refState!=null)&&(!refState.equals(""))) {
1818                                                                 State state = states.get(refState);
1819                                                                 fn.addInapplicableState(state);
1820                                                         }
1821                                                 }
1822                                         }
1823                                         Element elOnlyapplicableIf = elCharNode.getChild("OnlyApplicableIf", sddNamespace);
1824                                         if (elOnlyapplicableIf!=null){
1825                                                 List<Element> listStates = elInapplicableIf.getChildren("State", sddNamespace);
1826                                                 for (Element stateElement : listStates) {
1827                                                         String refState = stateElement.getAttributeValue("ref");
1828                                                         if ((refState!=null)&&(!refState.equals(""))) {
1829                                                                 State state = states.get(refState);
1830                                                                 fn.addApplicableState(state);
1831                                                         }
1832                                                 }
1833                                         }
1834                                 }
1835
1836                                 if (elParent!=null){
1837                                         String refP = elParent.getAttributeValue("ref");
1838                                         if ((refP!=null)&&(!refP.equals(""))) {
1839                                         FeatureNode parent = featureNodes.get(refP);
1840                                                 if (parent==null){
1841                                                 parent = root; // if no parent found or the reference is broken, add the node to the root of the tree
1842                                                 }
1843                                                 parent.addChild(fn);
1844                                         }
1845                                 }
1846                                 String refC = elCharacter.getAttributeValue("ref");
1847                                 if ((refC!=null)&&(!refC.equals(""))){
1848                                         Feature character = features.get(refC);
1849                                         fn.setFeature(character);
1850                                         featureNodes.put(refC, fn);
1851                                 }
1852                         }
1853                 }
1854         }
1855
1856         // imports the <TaxonHierarchies> block
1857         protected void importTaxonHierarchies(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1858
1859                 logger.info("start TaxonHierarchies ...");
1860                 Element elTaxonHierarchies = elDataset.getChild("TaxonHierarchies",sddNamespace);
1861
1862                 if (elTaxonHierarchies != null) {
1863                         List<Element> listTaxonHierarchies = elTaxonHierarchies.getChildren("TaxonHierarchy", sddNamespace);
1864                         int j = 0;
1865                         for (Element elTaxonHierarchy : listTaxonHierarchies){
1866                                 try {
1867                                         Element elRepresentation = elTaxonHierarchy.getChild("Representation",sddNamespace);
1868                                         String label = (String)ImportHelper.getXmlInputValue(elRepresentation,"Label",sddNamespace);
1869                                                 Classification classification =  Classification.NewInstance(label);
1870                                                 importRepresentation(elTaxonHierarchy, sddNamespace, classification, "", cdmState);
1871
1872                                                 List<TaxonNode> root = classification.getChildNodes();
1873                                                 Element elNodes = elTaxonHierarchy.getChild("Nodes", sddNamespace); // There can be only one <Nodes> block for TaxonHierarchies
1874                                                 List<Element> listNodes = elNodes.getChildren("Node", sddNamespace);
1875
1876                                                 for (Element elNode : listNodes){
1877                                                         String idN = elNode.getAttributeValue("id");
1878                                                         TaxonNameBase tnb = null;
1879                                                         if (!idN.equals("")) {
1880                                                                 Element elTaxonName = elNode.getChild("TaxonName", sddNamespace);
1881                                                                 String refTN = elTaxonName.getAttributeValue("ref");
1882                                                                 tnb = taxonNameBases.get(refTN);
1883                                                                 Taxon taxon = (Taxon) tnb.getTaxa().iterator().next() ;
1884                                                                 Element elParent = elNode.getChild("Parent", sddNamespace);
1885                                                                 if (elParent!=null){
1886                                                                         String refP = elParent.getAttributeValue("ref");
1887                                                                         if (!refP.equals("")) {
1888                                                                                 TaxonNode parent = taxonNodes.get(refP);
1889                                                                                 TaxonNode child = parent.addChildTaxon(taxon, sec, null);
1890                                                                                 child.setSynonymToBeUsed( Synonym.NewInstance(tnb, sec)); //TODO is this required??
1891                                                                                 taxonNodes.put(idN,child);
1892                                                                         }
1893                                                                 }
1894                                                                 else {
1895                                                                         TaxonNode tn = classification.addChildTaxon(taxon, sec, null); // if no parent found or the reference is broken, add the node to the root of the tree
1896                                                                         tn.setSynonymToBeUsed( Synonym.NewInstance(tnb, sec));  //TODO is this required??
1897                                                                         taxonNodes.put(idN,tn);
1898                                                                 }
1899                                                         }
1900                                                 }
1901
1902                                                 classifications.add(classification);
1903                                         }
1904
1905                                 catch (Exception e) {
1906                                         //FIXME
1907                                         logger.warn("Import of Taxon Hierarchy " + j + " failed.");
1908                                         cdmState.setUnsuccessfull();
1909                                 }
1910
1911                                 if ((++j % modCount) == 0){ logger.info("TaxonHierarchies handled: " + j);}
1912
1913                         }
1914
1915                 }
1916         }
1917
1918
1919         // imports the <GeographicAreas> block
1920         protected void importGeographicAreas(Element elDataset, Namespace sddNamespace, SDDImportState cdmState) {
1921                 Element elGeographicAreas = elDataset.getChild("GeographicAreas",sddNamespace);
1922                 if (elGeographicAreas != null) {
1923                         List<Element> listGeographicAreas = elGeographicAreas.getChildren("GeographicArea", sddNamespace);
1924                         int j = 0;
1925
1926                         for (Element elGeographicArea : listGeographicAreas){
1927
1928                                 String id = elGeographicArea.getAttributeValue("id");
1929                                 NamedArea na = new NamedArea();
1930                                 importRepresentation(elGeographicArea, sddNamespace, na, id, cdmState);
1931                                 namedAreas.put(id,na);
1932                                                                 }
1933                         if ((++j % modCount) == 0){ logger.info("GeographicAreas handled: " + j);}
1934                                                         }
1935                                                         }
1936 }