cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/sdd/in/SDDDataSetImport.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.sdd.in;
  11
  12 import java.io.File;
  13 import java.net.MalformedURLException;
  14 import java.net.URI;
  15 import java.net.URL;
  16 import java.text.SimpleDateFormat;
  17 import java.util.ArrayList;
  18 import java.util.Date;
  19 import java.util.HashMap;
  20 import java.util.HashSet;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Set;
  24
  25 import org.apache.commons.lang.StringUtils;
  26 import org.apache.log4j.Logger;
  27 import org.jdom.Element;
  28 import org.jdom.Namespace;
  29 import org.joda.time.DateTime;
  30 import org.springframework.stereotype.Component;
  31 import org.springframework.transaction.TransactionStatus;
  32
  33 import eu.etaxonomy.cdm.api.service.IDescriptionService;
  34 import eu.etaxonomy.cdm.common.media.ImageInfo;
  35 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  36 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  37 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  38 import eu.etaxonomy.cdm.io.common.ICdmIO;
  39 import eu.etaxonomy.cdm.io.common.ICdmImport;
  40 import eu.etaxonomy.cdm.io.common.ImportHelper;
  41 import eu.etaxonomy.cdm.io.sdd.SDDTransformer;
  42 import eu.etaxonomy.cdm.model.agent.Person;
  43 import eu.etaxonomy.cdm.model.agent.Team;
  44 import eu.etaxonomy.cdm.model.common.Annotation;
  45 import eu.etaxonomy.cdm.model.common.AnnotationType;
  46 import eu.etaxonomy.cdm.model.common.CdmBase;
  47 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  48 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  49 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  50 import eu.etaxonomy.cdm.model.common.Language;
  51 import eu.etaxonomy.cdm.model.common.LanguageString;
  52 import eu.etaxonomy.cdm.model.common.Marker;
  53 import eu.etaxonomy.cdm.model.common.MarkerType;
  54 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  55 import eu.etaxonomy.cdm.model.common.Representation;
  56 import eu.etaxonomy.cdm.model.common.TermBase;
  57 import eu.etaxonomy.cdm.model.common.TermVocabulary;
  58 import eu.etaxonomy.cdm.model.common.VersionableEntity;
  59 import eu.etaxonomy.cdm.model.description.CategoricalData;
  60 import eu.etaxonomy.cdm.model.description.Feature;
  61 import eu.etaxonomy.cdm.model.description.FeatureNode;
  62 import eu.etaxonomy.cdm.model.description.FeatureTree;
  63 import eu.etaxonomy.cdm.model.description.MeasurementUnit;
  64 import eu.etaxonomy.cdm.model.description.Modifier;
  65 import eu.etaxonomy.cdm.model.description.QuantitativeData;
  66 import eu.etaxonomy.cdm.model.description.State;
  67 import eu.etaxonomy.cdm.model.description.StateData;
  68 import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
  69 import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
  70 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  71 import eu.etaxonomy.cdm.model.description.TextData;
  72 import eu.etaxonomy.cdm.model.description.WorkingSet;
  73 import eu.etaxonomy.cdm.model.location.NamedArea;
  74 import eu.etaxonomy.cdm.model.media.IdentifiableMediaEntity;
  75 import eu.etaxonomy.cdm.model.media.ImageFile;
  76 import eu.etaxonomy.cdm.model.media.Media;
  77 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
  78 import eu.etaxonomy.cdm.model.media.MediaRepresentationPart;
  79 import eu.etaxonomy.cdm.model.media.Rights;
  80 import eu.etaxonomy.cdm.model.name.NonViralName;
  81 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  82 import eu.etaxonomy.cdm.model.occurrence.Specimen;
  83 import eu.etaxonomy.cdm.model.reference.Reference;
  84 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  85 import eu.etaxonomy.cdm.model.taxon.Classification;
  86 import eu.etaxonomy.cdm.model.taxon.Synonym;
  87 import eu.etaxonomy.cdm.model.taxon.Taxon;
  88 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  89
  90 /**
  91  * This class is not used yet. It is meant for refactoring the SDD import.
  92  *
  93  * @author a.mueller
  94  * @created 06.01.2011
  95  * @version 1.0
  96  */
  97 @Component
  98 public class SDDDataSetImport extends CdmImportBase<SDDImportConfigurator, SDDImportState> implements ICdmImport<SDDImportConfigurator, SDDImportState> {
  99         private static final Logger logger = Logger.getLogger(SDDDataSetImport.class);
 100
 101         private static int modCount = 1000;
 102
 103         private Map<String,Person> authors = new HashMap<String,Person>();
 104         private Map<String,String> citations = new HashMap<String,String>();
 105         private Map<String,String> defaultUnitPrefixes = new HashMap<String,String>();
 106         private Map<String,Person> editors = new HashMap<String,Person>();
 107         private Map<String,FeatureNode> featureNodes = new HashMap<String,FeatureNode>();
 108         private Map<String,Feature> features = new HashMap<String,Feature>();
 109         private Map<String,String> locations = new HashMap<String,String>();
 110         private Map<String,List<CdmBase>> mediaObject_ListCdmBase = new HashMap<String,List<CdmBase>>();
 111         private Map<String,String> mediaObject_Role = new HashMap<String,String>();
 112         private Map<String,Reference> publications = new HashMap<String,Reference>();
 113         private Map<String,State> states = new HashMap<String,State>();
 114         private Map<String,TaxonDescription> taxonDescriptions = new HashMap<String,TaxonDescription>();
 115         private Map<String,NonViralName> taxonNameBases = new HashMap<String,NonViralName>();
 116         private Map<String,MeasurementUnit> units = new HashMap<String,MeasurementUnit>();
 117         private Map<String,TaxonNode> taxonNodes = new HashMap<String,TaxonNode>();
 118         private Map<String,NamedArea> namedAreas = new HashMap<String,NamedArea>();
 119         private Map<String,Specimen> specimens = new HashMap<String,Specimen>();
 120         private Map<String,Modifier> modifiers = new HashMap<String,Modifier>();
 121
 122         private Set<MarkerType> markerTypes = new HashSet<MarkerType>();
 123         private Set<TermVocabulary> vocabularies = new HashSet<TermVocabulary>();
 124
 125         private Set<Feature> descriptiveConcepts = new HashSet<Feature>();
 126         private Set<AnnotationType> annotationTypes = new HashSet<AnnotationType>();
 127 //      private Set<Feature> featureSet = new HashSet<Feature>();
 128         private Set<Reference> sources = new HashSet<Reference>();
 129         private Reference sec = ReferenceFactory.newDatabase();
 130         private Reference sourceReference = null;
 131
 132         private Language datasetLanguage = null;
 133         private WorkingSet workingSet = null;
 134
 135         private Namespace xmlNamespace = Namespace.getNamespace("xml","http://www.w3.org/XML/1998/namespace");
 136
 137         private String generatorName = "";
 138         private String generatorVersion = "";
 139
 140
 141         private Set<StatisticalMeasure> statisticalMeasures = new HashSet<StatisticalMeasure>();
 142         private Set<VersionableEntity> featureData = new HashSet<VersionableEntity>();
 143         private Set<FeatureTree> featureTrees = new HashSet<FeatureTree>();
 144         private Set<Classification> classifications = new HashSet<Classification>();
 145
 146         private Rights copyright = null;
 147
 148         private int taxonNamesCount = 0;
 149
 150         protected Class<ICdmIO>[] ioClassList;
 151
 152         public SDDDataSetImport(){
 153                 super();
 154         }
 155
 156         protected void makeIoClassList(){
 157                 ioClassList = new Class[]{
 158                                 SDDImport.class
 159                 };
 160         };
 161
 162         @Override
 163         public boolean doCheck(SDDImportState state){
 164                 boolean result = true;
 165                 makeIoClassList();
 166
 167 //              for (Class<ICdmIO> ioClass: ioClassList){
 168 //                      try {
 169 //                              String ioBeanName = CdmApplicationAwareDefaultImport.getComponentBeanName(ioClass);
 170 //                              ICdmIO cdmIo = (ICdmIO)getTermService().getBean(ioBeanName, ICdmIO.class);
 171 //                              if (cdmIo != null){
 172 ////                                    result &= cdmIo.invoke(config, stores);
 173 //                                      state.setCurrentIO(cdmIo);
 174 //                                      result &= cdmIo.invoke(state);
 175 //                              }else{
 176 //                                      logger.error("cdmIO was null");
 177 //                                      result = false;
 178 //                              }
 179 //                      } catch (Exception e) {
 180 //                                      logger.error(e);
 181 //                                      e.printStackTrace();
 182 //                                      result = false;
 183 //                      }
 184 //              }
 185                 logger.warn("No check implemented for SDD");
 186                 return result;
 187         }
 188
 189         //      @Override
 190         //      public boolean doInvoke(IImportConfigurator config, Map<String, MapWrapper<? extends CdmBase>> stores){
 191         @Override
 192         public void doInvoke(SDDImportState state){
 193
 194                 TransactionStatus ts = startTransaction();
 195                 SDDImportConfigurator sddConfig = state.getConfig();
 196                 IProgressMonitor progressMonitor = sddConfig.getProgressMonitor();
 197
 198                 logger.info("start Datasets ...");
 199
 200                 // <Datasets xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://rs.tdwg.org/UBIF/2006/" xsi:schemaLocation="http://rs.tdwg.org/UBIF/2006/ ../SDD.xsd">
 201                 Element root = sddConfig.getSourceRoot();
 202                 Namespace sddNamespace = sddConfig.getSddNamespace();
 203
 204                 logger.info("start TechnicalMetadata ...");
 205                 // <TechnicalMetadata created="2006-04-20T10:00:00">
 206                 importTechnicalMetadata(root, sddNamespace, sddConfig);
 207                 List<Element> elDatasets = root.getChildren("Dataset",sddNamespace);
 208                 int i = 0;
 209
 210                 //for each Dataset
 211                 logger.info("start Dataset ...");
 212                 progressMonitor.beginTask("Importing SDD data", elDatasets.size());
 213                 for (Element elDataset : elDatasets){
 214                         importDataset(elDataset, sddNamespace, state);
 215 //                      if ((++i % modCount) == 0){ logger.info("dataset(s) handled: " + i);}
 216 //                      logger.info(i + " dataset(s) handled");
 217                         progressMonitor.worked(1);
 218                 }
 219                 commitTransaction(ts);
 220                 progressMonitor.done();
 221                 logger.info("End of transaction");
 222                 return;
 223         }
 224
 225         /* (non-Javadoc)
 226          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 227          */
 228         protected boolean isIgnore(SDDImportState state){
 229                 return false;
 230         }
 231
 232
 233         // associates the reference of a media object in SDD with a CdmBase Object
 234         protected void associateImageWithCdmBase(String refMO, CdmBase cb){
 235                 if ((refMO != null) && (cb!=null)) {
 236                         if (! refMO.equals("")) {
 237                                 if (! mediaObject_ListCdmBase.containsKey(refMO)) {
 238                                         List<CdmBase> lcb = new ArrayList<CdmBase>();
 239                                         lcb.add(cb);
 240                                         mediaObject_ListCdmBase.put(refMO,lcb);
 241                                 } else {
 242                                         List<CdmBase> lcb = mediaObject_ListCdmBase.get(refMO);
 243                                         lcb.add(cb);
 244                                         mediaObject_ListCdmBase.put(refMO,lcb);
 245                                 }
 246                         }
 247                 }
 248         }
 249
 250         // imports information about the Dataset
 251         protected void importDatasetRepresentation(Element parent, Namespace sddNamespace){
 252                 logger.info("start Representation ...");
 253                 /* <Representation>
 254                         <Label>The Genus Viola</Label>
 255                         <Detail>This is an example for a very simple SDD file, representing a single description with categorical, quantitative, and text character. Compare also the "Fragment*" examples, which contain more complex examples in the form of document fragments. Intended for version="SDD 1.1".</Detail>
 256                </Representation>
 257                  */
 258
 259
 260
 261                 Element elRepresentation = parent.getChild("Representation",sddNamespace);
 262                 String label = (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace);
 263                 String detail = (String)ImportHelper.getXmlInputValue(elRepresentation, "Detail",sddNamespace);
 264
 265                 //new
 266                 Representation representation = Representation.NewInstance(detail, label, null, datasetLanguage);
 267                 workingSet.addRepresentation(representation);
 268
 269
 270                 //old
 271 //              sec.setTitleCache(label, true);
 272 //
 273 //              if (detail != null) {
 274 //                      Annotation annotation = Annotation.NewInstance(detail, datasetLanguage);
 275 //                      annotation.setAnnotationType(AnnotationType.EDITORIAL());
 276 //                      sec.addAnnotation(annotation);
 277 //              }
 278
 279
 280                 List<Element> listMediaObjects = elRepresentation.getChildren("MediaObject",sddNamespace);
 281
 282                 for (Element elMediaObject : listMediaObjects) {
 283                         String ref = null;
 284                         String role = null;
 285                         if (elMediaObject != null) {
 286                                 ref = elMediaObject.getAttributeValue("ref");
 287                                 role = elMediaObject.getAttributeValue("role");
 288                         }
 289                         if (ref != null) {
 290                                 if (!ref.equals("")) {
 291                                         this.associateImageWithCdmBase(ref,sourceReference);
 292                                         this.associateImageWithCdmBase(ref,sec);
 293                                         mediaObject_Role.put(ref,role);
 294                                 }
 295                         }
 296                 }
 297         }
 298
 299         // imports the representation (label, detail, lang) of a particular SDD element
 300         protected void importRepresentation(Element parent, Namespace sddNamespace, VersionableEntity ve, String id, SDDImportState state){
 301                 Element elRepresentation = parent.getChild("Representation",sddNamespace);
 302
 303                 Map<Language,List<String>> langLabDet = new HashMap<Language,List<String>>();
 304
 305                 handleRepresentationLabels(sddNamespace, elRepresentation, langLabDet);
 306                 handleRepresentationDetails(sddNamespace, elRepresentation, langLabDet);
 307
 308                 if (ve instanceof TermBase) {
 309                         makeRepresentationForTerms((TermBase)ve, langLabDet);
 310                 }else if (ve instanceof Media) {
 311                         makeRepresentationForMedia((Media)ve, langLabDet);
 312                 }else if (ve instanceof IdentifiableEntity<?>) {
 313                         IdentifiableEntity<?> ie = (IdentifiableEntity<?>)ve;
 314                         makeRepresentationForIdentifiableEntity(sddNamespace, ie, elRepresentation, langLabDet);
 315                         if (ve instanceof IdentifiableMediaEntity<?>){
 316                                 makeRepresentationForIdentifiableMediaEntity(parent, sddNamespace, (IdentifiableMediaEntity<?>)ve);
 317                         }
 318                 }
 319
 320                 makeRepresentationMediaObjects(sddNamespace, ve, elRepresentation);//FIXME
 321
 322         }
 323
 324
 325         /**
 326          * Handles the "Detail" children of representations. Adds the result to the langLabDet.
 327          * @param sddNamespace
 328          * @param elRepresentation
 329          * @param langLabDet
 330          */
 331         private void handleRepresentationDetails(Namespace sddNamespace,
 332                         Element elRepresentation, Map<Language, List<String>> langLabDet) {
 333                 List<Element> listDetails = elRepresentation.getChildren("Detail",sddNamespace);
 334                 for (Element elDetail : listDetails){
 335                         Language language = getLanguage(elDetail);
 336                         String role = elDetail.getAttributeValue("role");
 337                         String detail = elDetail.getText();
 338                         List<String> labDet = langLabDet.get(language);
 339                         labDet.add(detail);
 340                         labDet.add(role);
 341                         langLabDet.put(language, labDet);
 342                 }
 343         }
 344
 345         /**
 346          * Handles the "Label" children of representations. Adds the result to the langLabDet.
 347          * @param sddNamespace
 348          * @param elRepresentation
 349          * @param langLabDet
 350          */
 351         private void handleRepresentationLabels(Namespace sddNamespace,
 352                                 Element elRepresentation, Map<Language, List<String>> langLabDet) {
 353                 // <Label xml:lang="la">Viola hederacea Labill.</Label>
 354                 List<Element> listLabels = elRepresentation.getChildren("Label",sddNamespace);
 355                 for (Element elLabel : listLabels){
 356                         Language language = getLanguage(elLabel);
 357                         String label = elLabel.getText();
 358                         List<String> labDet = new ArrayList<String>(3);
 359                         labDet.add(label);
 360                         langLabDet.put(language, labDet);
 361                 }
 362         }
 363
 364         /**
 365          *
 366          * @param ve
 367          * @param langLabDet
 368          */
 369         private void makeRepresentationForMedia(Media m, Map<Language, List<String>> langLabDet) {
 370                 for (Language lang : langLabDet.keySet()){
 371                         List<String> labDet = langLabDet.get(lang);
 372                         if (labDet.get(0) != null){
 373                                 m.putTitle(LanguageString.NewInstance(labDet.get(0), lang));
 374                         }
 375                         if (labDet.size()>1) {
 376                                 m.addDescription(labDet.get(1), lang);
 377                         }
 378                 }
 379         }
 380
 381         /**
 382          * Handles representations for terms. Adds one representation per language in langLabDet.
 383          *
 384          * @param ve
 385          * @param langLabDet
 386          */
 387         private void makeRepresentationForTerms(TermBase tb, Map<Language, List<String>> langLabDet) {
 388                         for (Language lang : langLabDet.keySet()){
 389                                 List<String> labDet = langLabDet.get(lang);
 390                                 if (labDet.size()>0){
 391                                         if (labDet.size()>1) {
 392                                                 tb.addRepresentation(Representation.NewInstance(labDet.get(1), labDet.get(0), labDet.get(0), lang));
 393                                         } else {
 394                                                 tb.addRepresentation(Representation.NewInstance(labDet.get(0), labDet.get(0), labDet.get(0), lang));
 395                                         }
 396                                 }
 397                         }
 398         }
 399
 400
 401         /**
 402          * Handles the "MediaObject" children of representations.
 403          * @param sddNamespace
 404          * @param ve
 405          * @param elRepresentation
 406          */
 407         private void makeRepresentationMediaObjects(Namespace sddNamespace,     VersionableEntity ve, Element elRepresentation) {
 408                 List <Element> listMediaObjects = elRepresentation.getChildren("MediaObject", sddNamespace);
 409                 for (Element elMediaObject : listMediaObjects) {
 410                         String ref = null;
 411                         //TODO
 412                         String role = null;
 413                         if (elMediaObject != null) {
 414                                 ref = elMediaObject.getAttributeValue("ref");
 415                                 role = elMediaObject.getAttributeValue("role");
 416                         }
 417                         if (StringUtils.isNotBlank(ref)) {
 418                                 if (ve instanceof TaxonDescription) {
 419                                         TaxonDescription td = (TaxonDescription) ve;
 420                                         if (td.getSources().size() > 0) {
 421                                                 this.associateImageWithCdmBase(ref, td.getSources().iterator().next().getCitation());
 422                                         } else {
 423                                                 Reference<?> descriptionSource = ReferenceFactory.newGeneric();
 424                                                 sources.add(descriptionSource);
 425                                                 //TODO type
 426                                                 td.addSource(OriginalSourceType.Unknown, null, null, descriptionSource, null);
 427                                                 this.associateImageWithCdmBase(ref,descriptionSource);
 428                                         }
 429                                 } else {
 430                                         this.associateImageWithCdmBase(ref,ve);
 431                                 }
 432                         }
 433                 }
 434         }
 435
 436         /**
 437          * Handles the "Links" element
 438          * @param parent
 439          * @param sddNamespace
 440          * @param ve
 441          */
 442         private void makeRepresentationForIdentifiableMediaEntity(Element parent,
 443                         Namespace sddNamespace, IdentifiableMediaEntity ime) {
 444                 Element elLinks = parent.getChild("Links",sddNamespace);
 445
 446                 if (elLinks != null) {
 447
 448                         //  <Link rel="Alternate" href="http://www.diversitycampus.net/people/hagedorn"/>
 449                         List<Element> listLinks = elLinks.getChildren("Link", sddNamespace);
 450                         Media link = Media.NewInstance();
 451                         MediaRepresentation mr = MediaRepresentation.NewInstance();
 452                         int k = 0;
 453                         //for each Link
 454                         for (Element elLink : listLinks){
 455
 456                                 try {
 457                                         //TODO
 458                                         String rel = elLink.getAttributeValue("rel");
 459                                         String href = elLink.getAttributeValue("href");
 460                                         URI uri = new URI(href);
 461                                         mr.addRepresentationPart(MediaRepresentationPart.NewInstance(uri, null));
 462                                         link.addRepresentation(mr);
 463                                         ime.addMedia(link);
 464
 465                                 } catch (Exception e) {
 466                                         //FIXME
 467                                         logger.warn("Import of Link " + k + " failed.");
 468                                 }
 469
 470                                 if ((++k % modCount) == 0){ logger.info("Links handled: " + k);}
 471
 472                         }
 473                 }
 474         }
 475
 476         /**
 477          * @param sddNamespace
 478          * @param ve
 479          * @param elRepresentation
 480          * @param langLabDet
 481          * @return
 482          */
 483         private void makeRepresentationForIdentifiableEntity(Namespace sddNamespace, IdentifiableEntity<?> ie,
 484                                         Element elRepresentation, Map<Language, List<String>> langLabDet) {
 485                 List<String> labDet = null;
 486
 487                 if (ie instanceof TaxonNameBase) {
 488                         if (langLabDet.keySet().contains(getTermService().getLanguageByIso("la"))) {
 489                                 labDet = langLabDet.get(getTermService().getLanguageByIso("la"));
 490                         } else if (langLabDet.keySet().contains(datasetLanguage)) {
 491                                 labDet = langLabDet.get(datasetLanguage);
 492                                 logger.info("TaxonName " + (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace) + " is not specified as a latin name.");
 493                         } else {
 494                                 labDet = langLabDet.get(langLabDet.keySet().iterator().next());
 495                                 logger.info("TaxonName " + (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace) + " is not specified as a latin name.");
 496                         }
 497                 } else {
 498                         labDet = langLabDet.get(langLabDet.keySet().iterator().next());
 499                 }
 500
 501                 //FIXME labDet is != null only for TaxonNameBase
 502                 ie.setTitleCache(labDet.get(0), true);
 503
 504                 if (labDet.size()>1) {
 505                         Annotation annotation = null;
 506                         if (labDet.get(1) != null) {
 507                                 if (labDet.get(2) != null) {
 508                                         annotation = Annotation.NewInstance(labDet.get(2) + " - " + labDet.get(1), datasetLanguage);
 509                                 } else {
 510                                         annotation = Annotation.NewInstance(labDet.get(1), datasetLanguage);
 511                                 }
 512                         }
 513                         ie.addAnnotation(annotation);
 514                 }
 515                 return;
 516         }
 517
 518         /**
 519          * @param elLabel
 520          * @return
 521          */
 522         private Language getLanguage(Element elLanguage) {
 523                 String lang = elLanguage.getAttributeValue("lang",xmlNamespace);
 524                 Language language = null;
 525                 if (StringUtils.isNotBlank(lang)) {
 526                         language = getTermService().getLanguageByIso(lang.substring(0, 2));
 527                 } else {
 528                         language = datasetLanguage;
 529                 }
 530                 return language;
 531         }
 532
 533
 534         // imports the representation (label, detail, lang) of a particular SDD element
 535         protected void importTechnicalMetadata(Element root, Namespace sddNamespace, SDDImportConfigurator sddConfig){
 536                 Element elTechnicalMetadata = root.getChild("TechnicalMetadata", sddNamespace);
 537                 String nameCreated = elTechnicalMetadata.getAttributeValue("created");
 538                 sourceReference = sddConfig.getSourceReference();
 539
 540                 if (nameCreated != null) {
 541                         if (!nameCreated.equals("")) {
 542                                 int year = Integer.parseInt(nameCreated.substring(0,4));
 543                                 int monthOfYear = Integer.parseInt(nameCreated.substring(5,7));
 544                                 int dayOfMonth = Integer.parseInt(nameCreated.substring(8,10));
 545                                 int hourOfDay = Integer.parseInt(nameCreated.substring(11,13));
 546                                 int minuteOfHour = Integer.parseInt(nameCreated.substring(14,16));
 547                                 int secondOfMinute = Integer.parseInt(nameCreated.substring(17,19));
 548                                 DateTime created = new DateTime(year,monthOfYear,dayOfMonth,hourOfDay,minuteOfHour,secondOfMinute,0);
 549                                 sourceReference.setCreated(created);
 550                                 sec.setCreated(created);
 551                         }
 552                 }
 553
 554                 // <Generator name="n/a, handcrafted instance document" version="n/a"/>
 555                 Element elGenerator = elTechnicalMetadata.getChild("Generator", sddNamespace);
 556                 generatorName = elGenerator.getAttributeValue("name");
 557                 generatorVersion = elGenerator.getAttributeValue("version");
 558
 559                 sec.addAnnotation(Annotation.NewDefaultLanguageInstance(generatorName + " - " + generatorVersion));
 560                 sourceReference.addAnnotation(Annotation.NewDefaultLanguageInstance(generatorName + " - " + generatorVersion));
 561
 562         }
 563
 564         // imports the complete dataset information
 565         protected void importDataset(Element elDataset, Namespace sddNamespace, SDDImportState state){                  // <Dataset xml:lang="en-us">
 566 //              SDDImportConfigurator sddConfig = state.getConfig();
 567
 568                 workingSet = WorkingSet.NewInstance();
 569                 importDatasetLanguage(elDataset,state);
 570                 importDatasetRepresentation(elDataset, sddNamespace);
 571                 importRevisionData(elDataset, sddNamespace);
 572                 importIPRStatements(elDataset, sddNamespace, state);
 573                 importTaxonNames(elDataset, sddNamespace, state);
 574
 575                 importDescriptiveConcepts(elDataset, sddNamespace, state);
 576                 importCharacters(elDataset, sddNamespace, state);
 577                 importCharacterTrees(elDataset, sddNamespace, state);
 578
 579                 MarkerType editorMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerEditor, "editor", "Editor", "edt");
 580                 MarkerType geographicAreaMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerSDDGeographicArea, "SDDGeographicArea", "SDDGeographicArea", "ga");
 581                 MarkerType descriptiveConceptMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerDescriptiveConcept, "DescriptiveConcept", "Descriptive Concept", "DC");
 582                 markerTypes.add(editorMarkerType);
 583                 markerTypes.add(geographicAreaMarkerType);
 584                 markerTypes.add(descriptiveConceptMarkerType);
 585
 586                 //saving of all imported data into the CDM db
 587                 saveVocabularies();
 588                 saveFeatures();
 589                 saveModifiers();
 590                 saveStates();
 591                 saveMarkerType();
 592                 saveAreas(geographicAreaMarkerType);
 593                 saveUnits();
 594                 saveStatisticalMeasure();
 595                 saveAnnotationType();
 596
 597                 importCodedDescriptions(elDataset, sddNamespace, state);
 598                 importAgents(elDataset, sddNamespace, state);
 599                 importPublications(elDataset, sddNamespace, state);
 600                 importMediaObjects(elDataset, sddNamespace, state);
 601                 importTaxonHierarchies(elDataset, sddNamespace, state);
 602                 importGeographicAreas(elDataset, sddNamespace, state);
 603                 importSpecimens(elDataset,sddNamespace, state);
 604
 605
 606                 if ((authors != null)||(editors != null)) {
 607                         Team team = Team.NewInstance();
 608                         if (authors != null) {
 609                                 for (Person author : authors.values()){
 610                                         team.addTeamMember(author);
 611                                 }
 612                         }
 613                         if (editors != null) {
 614                                 Marker marker = Marker.NewInstance();
 615                                 marker.setMarkerType(editorMarkerType);
 616                                 for (Person editor : editors.values()){
 617                                         Person edit = editor;
 618                                         edit.addMarker(marker);
 619                                         team.addTeamMember(edit);
 620                                 }
 621                         }
 622                         sec.setAuthorTeam(team);
 623                         sourceReference.setAuthorTeam(team);
 624                 }
 625
 626                 if (copyright != null) {
 627                         sourceReference.addRights(copyright);
 628                         sec.addRights(copyright);
 629                 }
 630
 631                 // Returns a CdmApplicationController created by the values of this configuration.
 632                 IDescriptionService descriptionService = getDescriptionService();
 633
 634                 for (TaxonDescription taxonDescription : taxonDescriptions.values()){
 635                         // Persists a Description
 636                         descriptionService.save(taxonDescription);
 637                 }
 638
 639                 for (String ref : taxonDescriptions.keySet()){
 640                         TaxonDescription td = taxonDescriptions.get(ref);
 641                         if (citations.containsKey(ref)) {
 642                                 Reference<?> publication = publications.get(citations.get(ref));
 643                                 if (locations.containsKey(ref)) {
 644                                         Annotation location = Annotation.NewInstance(locations.get(ref), datasetLanguage);
 645                                         AnnotationType annotationType = AnnotationType.NewInstance("", "location", "");
 646                                         annotationTypes.add(annotationType);
 647                                         location.setAnnotationType(annotationType);
 648                                         (publication).addAnnotation(location);
 649                                 }
 650                                 //TODO type
 651                                 td.addSource(OriginalSourceType.Unknown, null, null, publication, null);
 652                         }
 653                 }
 654                 logger.info("end makeTaxonDescriptions ...");
 655
 656                 if (descriptiveConcepts != null) {
 657                         for (Feature feature : descriptiveConcepts) {
 658                                 Marker marker = Marker.NewInstance();
 659                                 marker.setMarkerType(descriptiveConceptMarkerType);
 660                                 feature.addMarker(marker);
 661                         }
 662                 }
 663                 saveFeatures();
 664
 665                 for (Reference publication : publications.values()){
 666                         getReferenceService().save(publication);
 667                 }
 668
 669                 for (Reference source : sources){
 670                         getReferenceService().save(source);
 671                 }
 672
 673                 for (FeatureTree featureTree : featureTrees) {
 674                         getFeatureTreeService().save(featureTree);
 675                 }
 676                 getWorkingSetService().save(workingSet);
 677                 for (Classification classification : classifications) {
 678                         getClassificationService().save(classification);
 679                 }
 680                 for (Specimen specimen : specimens.values()) {
 681                         getOccurrenceService().save(specimen);
 682                 }
 683                 logger.info("end of persistence ...");
 684
 685                 return;
 686         }
 687
 688         /**
 689          *
 690          */
 691         private void saveVocabularies() {
 692                 for (TermVocabulary vocabulary : vocabularies ){
 693                         getVocabularyService().save(vocabulary);
 694                 }
 695
 696         }
 697
 698         private void saveAnnotationType() {
 699                 for (AnnotationType annotationType: annotationTypes){
 700                         getTermService().save(annotationType);
 701                 }
 702         }
 703
 704         private void saveStatisticalMeasure() {
 705                 for (StatisticalMeasure sm : statisticalMeasures){
 706                         getTermService().save(sm);
 707                 }
 708         }
 709
 710         private void saveUnits() {
 711                 if (units != null) {
 712                         for (MeasurementUnit unit : units.values()){
 713                                 if (unit != null) {
 714                                         getTermService().save(unit);
 715                                 }
 716                         }
 717                 }
 718         }
 719
 720         private void saveAreas(MarkerType geographicAreaMarkerType) {
 721                 for (NamedArea area : namedAreas.values() ){
 722                         Marker marker = Marker.NewInstance();
 723                         marker.setMarkerType(geographicAreaMarkerType);
 724                         area.addMarker(marker);
 725                         getTermService().save(area);
 726                 }
 727         }
 728
 729         private void saveStates() {
 730                 for (State state : states.values() ){
 731                         getTermService().save(state);
 732                 }
 733         }
 734
 735         private void saveMarkerType() {
 736                 for (MarkerType markerType : markerTypes){
 737                         getTermService().save(markerType);
 738                 }
 739         }
 740
 741         private void saveModifiers() {
 742                 for (Modifier modifier : modifiers.values() ){
 743                         getTermService().save(modifier);
 744                 }
 745         }
 746
 747         private void saveFeatures() {
 748                 for (Feature feature : features.values() ){
 749                         getTermService().save(feature);
 750                 }
 751         }
 752
 753         // imports the default language of the dataset
 754         protected void importDatasetLanguage(Element elDataset, SDDImportState state){
 755                 String nameLang = elDataset.getAttributeValue("lang",xmlNamespace);
 756
 757                 if (StringUtils.isNotBlank(nameLang)) {
 758                         String iso = nameLang.substring(0, 2);
 759                         datasetLanguage = getTermService().getLanguageByIso(iso);
 760                 } else {
 761                         datasetLanguage = Language.DEFAULT();
 762                 }
 763                 if (datasetLanguage == null) {
 764                         datasetLanguage = Language.DEFAULT();
 765                 }
 766         }
 767
 768         // imports the specimens
 769         protected void importSpecimens(Element elDataset, Namespace sddNamespace, SDDImportState cdmState) {
 770                 logger.info("start Specimens ...");
 771                 /*      <Specimens>
 772                         <Specimen id="sp1">
 773                                 <Representation>
 774                                 <Label>TJM45337</Label>
 775                                 </Representation>
 776                         </Specimen>
 777                 </Specimens>
 778                  */
 779                 Element elSpecimens = elDataset.getChild("Specimens",sddNamespace);
 780                 if (elSpecimens != null){
 781                         List<Element> listSpecimens = elSpecimens.getChildren("Specimen", sddNamespace);
 782                         int j = 0;
 783                         for (Element elSpecimen : listSpecimens) {
 784                                 String id = elSpecimen.getAttributeValue("id");
 785                                 Specimen specimen = null;
 786                                 if (!id.equals("")) {
 787                                         specimen = Specimen.NewInstance();
 788                                         specimens.put(id,specimen);
 789                                         importRepresentation(elSpecimen, sddNamespace, specimen, id, cdmState);
 790                                 }
 791                         }
 792
 793                 }
 794         }
 795
 796         // imports the revision data associated with the Dataset (authors, modifications)
 797         protected void importRevisionData(Element elDataset, Namespace sddNamespace){
 798                 // <RevisionData>
 799                 logger.info("start RevisionData ...");
 800                 Element elRevisionData = elDataset.getChild("RevisionData",sddNamespace);
 801                 if (elRevisionData != null){
 802                         // <Creators>
 803                         Element elCreators = elRevisionData.getChild("Creators",sddNamespace);
 804
 805                         // <Agent role="aut" ref="a1"/>
 806                         List<Element> listAgents = elCreators.getChildren("Agent", sddNamespace);
 807
 808                         int j = 0;
 809                         //for each Agent
 810                         for (Element elAgent : listAgents){
 811
 812                                 String role = elAgent.getAttributeValue("role");
 813                                 String ref = elAgent.getAttributeValue("ref");
 814                                 if (role.equals("aut")) {
 815                                         if(!ref.equals("")) {
 816                                                 authors.put(ref, null);
 817                                         }
 818                                 }
 819                                 if (role.equals("edt")) {
 820                                         if(!ref.equals("")) {
 821                                                 editors.put(ref, null);
 822                                         }
 823                                 }
 824                                 if ((++j % modCount) == 0){ logger.info("Agents handled: " + j);}
 825
 826                         }
 827
 828                         // <DateModified>2006-04-08T00:00:00</DateModified>
 829                         String stringDateModified = (String)ImportHelper.getXmlInputValue(elRevisionData, "DateModified",sddNamespace);
 830
 831                         if (stringDateModified != null) {
 832                                 SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss");
 833                                 Date d = null;
 834                                 try {
 835                                         d = sdf.parse(stringDateModified);
 836                                 } catch(Exception e) {
 837                                         System.err.println("Exception :");
 838                                         e.printStackTrace();
 839                                 }
 840
 841                                 DateTime updated = null;
 842                                 if (d != null) {
 843                                         updated = new DateTime(d);
 844                                         sourceReference.setUpdated(updated);
 845                                         sec.setUpdated(updated);
 846                                 }
 847                         }
 848                 }
 849         }
 850
 851         // imports ipr statements associated with a dataset
 852         protected void importIPRStatements(Element elDataset, Namespace sddNamespace, SDDImportState state){
 853                 // <IPRStatements>
 854                 logger.info("start IPRStatements ...");
 855                 Element elIPRStatements = elDataset.getChild("IPRStatements",sddNamespace);
 856                 // <IPRStatement role="Copyright">
 857                 if (elIPRStatements != null) {
 858                         List<Element> listIPRStatements = elIPRStatements.getChildren("IPRStatement", sddNamespace);
 859                         int j = 0;
 860                         //for each IPRStatement
 861
 862                         for (Element elIPRStatement : listIPRStatements){
 863
 864                                 String role = elIPRStatement.getAttributeValue("role");
 865                                 // <Label xml:lang="en-au">(c) 2003-2006 Centre for Occasional Botany.</Label>
 866                                 Element elLabel = elIPRStatement.getChild("Label",sddNamespace);
 867                                 String lang = "";
 868                                 if (elLabel != null) {
 869                                         lang = elLabel.getAttributeValue("lang",xmlNamespace);
 870                                 }
 871                                 String label = (String)ImportHelper.getXmlInputValue(elIPRStatement, "Label",sddNamespace);
 872
 873                                 if (role.equals("Copyright")) {
 874                                         Language iprLanguage = null;
 875                                         if (lang != null) {
 876                                                 if (!lang.equals("")) {
 877                                                         iprLanguage = getTermService().getLanguageByIso(lang.substring(0, 2));
 878                                                 } else {
 879                                                         iprLanguage = datasetLanguage;
 880                                                 }
 881                                         }
 882                                         if (iprLanguage == null) {
 883                                                 iprLanguage = datasetLanguage;
 884                                         }
 885                                         copyright = Rights.NewInstance(label, iprLanguage);
 886                                 }
 887
 888                                 if (copyright != null) {
 889                                         sourceReference.addRights(copyright);
 890                                         sec.addRights(copyright);
 891                                 }
 892
 893                                 if ((++j % modCount) == 0){ logger.info("IPRStatements handled: " + j);}
 894
 895                         }
 896                 }
 897         }
 898
 899         // imports the taxon names
 900         protected void importTaxonNames(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
 901                 // <TaxonNames>
 902                 logger.info("start TaxonNames ...");
 903                 Element elTaxonNames = elDataset.getChild("TaxonNames",sddNamespace);
 904                 // <TaxonName id="t1" uri="urn:lsid:authority:namespace:my-own-id">
 905                 if (elTaxonNames != null) {
 906                         List<Element> listTaxonNames = elTaxonNames.getChildren("TaxonName", sddNamespace);
 907                         int j = 0;
 908                         //for each TaxonName
 909                         for (Element elTaxonName : listTaxonNames){
 910
 911                                 String id = elTaxonName.getAttributeValue("id");
 912                                 String uri = elTaxonName.getAttributeValue("uri");
 913
 914                                 NonViralName tnb = null;
 915                                 if (!id.equals("")) {
 916                                         tnb = NonViralName.NewInstance(null);
 917                                         IdentifiableSource source = null;
 918                                         if (isNotBlank(uri)) {
 919                                                 //TODO type
 920                                                 source = IdentifiableSource.NewInstance(OriginalSourceType.Unknown, id, "TaxonName", ReferenceFactory.newGeneric(), uri);
 921                                         } else {
 922                                                 source = IdentifiableSource.NewDataImportInstance(id, "TaxonName");
 923                                         }
 924                                         tnb.addSource(source);
 925                                         taxonNameBases.put(id,tnb);
 926                                 }
 927
 928                                 // <Representation>
 929                                 // <Label xml:lang="la">Viola hederacea Labill.</Label>
 930                                 importRepresentation(elTaxonName, sddNamespace, tnb, id, cdmState);
 931
 932                                 if ((++j % modCount) == 0){ logger.info("TaxonNames handled: " + j);}
 933
 934                         }
 935                 }
 936         }
 937
 938         // imports the characters (categorical, quantitative and text ; sequence characters not supported) which correspond to CDM Features
 939         protected void importCharacters(Element elDataset, Namespace sddNamespace, SDDImportState state){
 940                 // <Characters>
 941                 logger.info("start Characters ...");
 942                 Element elCharacters = elDataset.getChild("Characters", sddNamespace);
 943
 944                 // <CategoricalCharacter id="c1">
 945                 if (elCharacters != null) {
 946                         handleCategoricalData(sddNamespace, state, elCharacters);
 947                         handleQuantitativeData(sddNamespace, state, elCharacters);
 948                         handleTextCharacters(sddNamespace, state, elCharacters);
 949                 }
 950
 951                 /*for (Iterator<Feature> f = features.values().iterator() ; f.hasNext() ;){
 952                         featureSet.add(f.next()); //XIM Why this line ?
 953                 }*/
 954
 955                 return;
 956
 957         }
 958
 959         /**
 960          * @param sddNamespace
 961          * @param cdmState
 962          * @param elCharacters
 963          * @return
 964          */
 965         private void handleCategoricalData(Namespace sddNamespace, SDDImportState cdmState, Element elCharacters) {
 966                 List<Element> elCategoricalCharacters = elCharacters.getChildren("CategoricalCharacter", sddNamespace);
 967                 int j = 0;
 968                 for (Element elCategoricalCharacter : elCategoricalCharacters){
 969                         try {
 970
 971                                 String idCC = elCategoricalCharacter.getAttributeValue("id");
 972                                 Feature categoricalCharacter = Feature.NewInstance();
 973                                 categoricalCharacter.setKindOf(Feature.DESCRIPTION());
 974                                 importRepresentation(elCategoricalCharacter, sddNamespace, categoricalCharacter, idCC, cdmState);
 975                                 categoricalCharacter.setSupportsCategoricalData(true);
 976
 977                                 // <States>
 978                                 Element elStates = elCategoricalCharacter.getChild("States",sddNamespace);
 979
 980                                 // <StateDefinition id="s1">
 981                                 List<Element> elStateDefinitions = elStates.getChildren("StateDefinition",sddNamespace);
 982                                 TermVocabulary<State> termVocabularyState = TermVocabulary.NewInstance(null, null, null, null);
 983
 984                                 vocabularies.add(termVocabularyState);
 985
 986                                 int k = 0;
 987                                 //for each StateDefinition
 988                                 for (Element elStateDefinition : elStateDefinitions){
 989
 990                                         if ((++k % modCount) == 0){ logger.info("StateDefinitions handled: " + (k-1));}
 991
 992                                         String idS = elStateDefinition.getAttributeValue("id");
 993                                         State state = states.get(idS);
 994                                         if (state == null){
 995                                                 state = State.NewInstance();
 996                                         }else{
 997                                                 logger.debug("State duplicate found");
 998                                         }
 999                                         importRepresentation(elStateDefinition, sddNamespace, state, idS, cdmState);
1000
1001                                         termVocabularyState.addTerm(state);
1002                                         states.put(idS,state);
1003                                 }
1004                                 categoricalCharacter.addSupportedCategoricalEnumeration(termVocabularyState);
1005                                 features.put(idCC, categoricalCharacter);
1006
1007                         } catch (Exception e) {
1008                                 logger.warn("Import of CategoricalCharacter " + j + " failed.");
1009                                 cdmState.setUnsuccessfull();
1010                         }
1011
1012                         if ((++j % modCount) == 0){ logger.info("CategoricalCharacters handled: " + j);}
1013
1014                 }
1015                 return;
1016         }
1017
1018         /**
1019          * @param sddNamespace
1020          * @param sddConfig
1021          * @param elCharacters
1022          */
1023         private void handleQuantitativeData(Namespace sddNamespace,     SDDImportState cdmState, Element elCharacters) {
1024                 int j;
1025                 // <QuantitativeCharacter id="c2">
1026                 List<Element> elQuantitativeCharacters = elCharacters.getChildren("QuantitativeCharacter", sddNamespace);
1027                 j = 0;
1028                 //for each QuantitativeCharacter
1029                 for (Element elQuantitativeCharacter : elQuantitativeCharacters){
1030
1031                         try {
1032
1033                                 String idQC = elQuantitativeCharacter.getAttributeValue("id");
1034
1035                                 // <Representation>
1036                                 //  <Label>Leaf length</Label>
1037                                 // </Representation>
1038                                 Feature quantitativeCharacter = Feature.NewInstance();
1039                                 quantitativeCharacter.setKindOf(Feature.DESCRIPTION());
1040                                 importRepresentation(elQuantitativeCharacter, sddNamespace, quantitativeCharacter, idQC, cdmState);
1041
1042                                 quantitativeCharacter.setSupportsQuantitativeData(true);
1043
1044                                 // <MeasurementUnit>
1045                                 //  <Label role="Abbrev">m</Label>
1046                                 // </MeasurementUnit>
1047                                 Element elMeasurementUnit = elQuantitativeCharacter.getChild("MeasurementUnit",sddNamespace);
1048                                 String label = "";
1049                                 String role = "";
1050                                 if (elMeasurementUnit != null) {
1051                                         Element elLabel = elMeasurementUnit.getChild("Label",sddNamespace);
1052                                         role = elLabel.getAttributeValue("role");
1053                                         label = (String)ImportHelper.getXmlInputValue(elMeasurementUnit, "Label",sddNamespace);
1054                                 }
1055
1056                                 MeasurementUnit unit = null;
1057                                 if (!label.equals("")){
1058                                         if (role != null) {
1059                                                 if (role.equals("Abbrev")){
1060                                                         unit = MeasurementUnit.NewInstance(label,label,label);
1061                                                 }
1062                                         } else {
1063                                                 unit = MeasurementUnit.NewInstance(label,label,label);
1064                                         }
1065                                 }
1066
1067                                 if (unit != null) {
1068                                         units.put(idQC, unit);
1069                                 }
1070
1071                                 //<Default>
1072                                 //  <MeasurementUnitPrefix>milli</MeasurementUnitPrefix>
1073                                 //</Default>
1074                                 Element elDefault = elQuantitativeCharacter.getChild("Default",sddNamespace);
1075                                 if (elDefault != null) {
1076                                         String measurementUnitPrefix = (String)ImportHelper.getXmlInputValue(elDefault, "MeasurementUnitPrefix",sddNamespace);
1077                                         if (! measurementUnitPrefix.equals("")){
1078                                                 defaultUnitPrefixes.put(idQC, measurementUnitPrefix);
1079                                         }
1080                                 }
1081
1082                                 features.put(idQC, quantitativeCharacter);
1083
1084                         } catch (Exception e) {
1085                                 //FIXME
1086                                 logger.warn("Import of QuantitativeCharacter " + j + " failed.");
1087                                 cdmState.setUnsuccessfull();
1088                         }
1089
1090                         if ((++j % modCount) == 0){ logger.info("QuantitativeCharacters handled: " + j);}
1091
1092                 }
1093                 return;
1094         }
1095
1096         private void handleTextCharacters(Namespace sddNamespace, SDDImportState cdmState, Element elCharacters) {
1097                 int j;
1098                 // <TextCharacter id="c3">
1099                 List<Element> elTextCharacters = elCharacters.getChildren("TextCharacter", sddNamespace);
1100                 j = 0;
1101                 //for each TextCharacter
1102                 for (Element elTextCharacter : elTextCharacters){
1103
1104                         try {
1105
1106                                 String idTC = elTextCharacter.getAttributeValue("id");
1107
1108                                 // <Representation>
1109                                 //  <Label xml:lang="en">Leaf features not covered by other characters</Label>
1110                                 // </Representation>
1111                                 Feature textCharacter = Feature.NewInstance();
1112                                 textCharacter.setKindOf(Feature.DESCRIPTION());
1113                                 importRepresentation(elTextCharacter, sddNamespace, textCharacter, idTC, cdmState);
1114
1115                                 textCharacter.setSupportsTextData(true);
1116
1117                                 features.put(idTC, textCharacter);
1118
1119                         } catch (Exception e) {
1120                                 //FIXME
1121                                 logger.warn("Import of TextCharacter " + j + " failed.");
1122                                 cdmState.setUnsuccessfull();
1123                         }
1124
1125                         if ((++j % modCount) == 0){ logger.info("TextCharacters handled: " + j);}
1126
1127                 }
1128                 return;
1129         }
1130
1131         // imports the descriptions of taxa
1132         protected void importCodedDescriptions(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1133
1134                 // <CodedDescriptions>
1135                 logger.info("start CodedDescriptions ...");
1136                 Element elCodedDescriptions = elDataset.getChild("CodedDescriptions",sddNamespace);
1137
1138                 // <CodedDescription id="D101">
1139                 if (elCodedDescriptions != null) {
1140                         List<Element> listCodedDescriptions = elCodedDescriptions.getChildren("CodedDescription", sddNamespace);
1141                         int j = 0;
1142                         //for each CodedDescription
1143                         for (Element elCodedDescription : listCodedDescriptions){
1144                                 handleCodedDescription(sddNamespace, cdmState, elCodedDescription, j);
1145                                 if ((++j % modCount) == 0){ logger.info("CodedDescriptions handled: " + j);}
1146                         }
1147                 }
1148                 return;
1149         }
1150
1151         /**
1152          * @param sddNamespace
1153          * @param sddConfig
1154          * @param j
1155          * @param elCodedDescription
1156          * @return
1157          */
1158         private void handleCodedDescription(Namespace sddNamespace, SDDImportState cdmState, Element elCodedDescription, int j) {
1159                 try {
1160
1161                         String idCD = elCodedDescription.getAttributeValue("id");
1162
1163                         // <Representation>
1164                         //  <Label>&lt;i&gt;Viola hederacea&lt;/i&gt; Labill. as revised by R. Morris April 8, 2006</Label>
1165                         // </Representation>
1166                         TaxonDescription taxonDescription = TaxonDescription.NewInstance();
1167                         if (!generatorName.isEmpty()){
1168                                 Annotation annotation = Annotation.NewInstance(generatorName, AnnotationType.TECHNICAL(),Language.DEFAULT());
1169                                 taxonDescription.addAnnotation(annotation);
1170                         }
1171                         importRepresentation(elCodedDescription, sddNamespace, taxonDescription, idCD, cdmState);
1172
1173                         // <Scope>
1174                         //  <TaxonName ref="t1"/>
1175                         //  <Citation ref="p1" location="p. 30"/>
1176                         // </Scope>
1177                         Element elScope = elCodedDescription.getChild("Scope", sddNamespace);
1178                         Taxon taxon;
1179                         if (elScope != null) {
1180                                 taxon = handleCDScope(sddNamespace, cdmState, idCD, elScope);
1181                         } else {//in case no taxon is linked to the description, a new one is created
1182                                 taxon = handleCDNoScope(sddNamespace, cdmState, elCodedDescription);
1183                         }
1184
1185                         // <SummaryData>
1186                         Element elSummaryData = elCodedDescription.getChild("SummaryData",sddNamespace);
1187                         if (elSummaryData != null) {
1188                                 handleSummaryCategoricalData(sddNamespace, taxonDescription, elSummaryData);
1189                                 handleSummaryQuantitativeData(sddNamespace, taxonDescription, elSummaryData);
1190                                 handleSummaryTextData(sddNamespace, taxonDescription, elSummaryData);
1191                         }
1192
1193                         if (taxon != null) {
1194                                 taxon.addDescription(taxonDescription);
1195                         }
1196 //
1197                         workingSet.addDescription(taxonDescription);
1198
1199 //OLD                   taxonDescription.setDescriptiveSystem(featureSet);
1200
1201                         taxonDescriptions.put(idCD, taxonDescription);//FIXME
1202
1203                 } catch (Exception e) {
1204                         //FIXME
1205                         logger.warn("Import of CodedDescription " + j + " failed.", e);
1206                         cdmState.setUnsuccessfull();
1207                 }
1208                 return;
1209         }
1210
1211         /**
1212          * @param sddNamespace
1213          * @param sddConfig
1214          * @param elCodedDescription
1215          * @param taxon
1216          * @return
1217          */
1218         private Taxon handleCDNoScope(Namespace sddNamespace,
1219                         SDDImportState cdmState, Element elCodedDescription     ) {
1220                 Taxon taxon = null;
1221                 NonViralName<?> nonViralName = NonViralName.NewInstance(null);
1222                 String id = new String("" + taxonNamesCount);
1223                 IdentifiableSource source = IdentifiableSource.NewDataImportInstance( id, "TaxonName");
1224                 importRepresentation(elCodedDescription, sddNamespace, nonViralName, id, cdmState);
1225
1226                 if(cdmState.getConfig().isDoMatchTaxa()){
1227                         taxon = getTaxonService().findBestMatchingTaxon(nonViralName.getTitleCache());
1228                 }
1229
1230                 if(taxon != null){
1231                         nonViralName = HibernateProxyHelper.deproxy(taxon.getName(), NonViralName.class);
1232 //                                                      taxonNameBases.put(id ,tnb);
1233 //                                                      taxonNamesCount++;
1234                         logger.info("using existing Taxon " + taxon.getTitleCache());
1235                 } else {
1236                         nonViralName.addSource(source);
1237                         taxonNameBases.put(id ,nonViralName);
1238                         taxonNamesCount++;
1239                         logger.info("creating new Taxon from TaxonName " + nonViralName.getTitleCache());
1240                         taxon = Taxon.NewInstance(nonViralName, sec);
1241                 }
1242                 return taxon;
1243         }
1244
1245         /**
1246          * @param sddNamespace
1247          * @param sddConfig
1248          * @param idCD
1249          * @param elScope
1250          * @param taxon
1251          * @return
1252          */
1253         private Taxon handleCDScope(Namespace sddNamespace, SDDImportState cdmState,
1254                         String idCD, Element elScope) {
1255                 Taxon taxon = null;
1256                 Element elTaxonName = elScope.getChild("TaxonName", sddNamespace);
1257                 String ref = elTaxonName.getAttributeValue("ref");
1258                 NonViralName nonViralName = taxonNameBases.get(ref);
1259
1260                 if(cdmState.getConfig().isDoMatchTaxa()){
1261                         taxon = getTaxonService().findBestMatchingTaxon(nonViralName.getTitleCache());
1262                 }
1263
1264                 if(taxon != null){
1265                         logger.info("using existing Taxon" + taxon.getTitleCache());
1266                         if(!nonViralName.getUuid().equals(taxon.getName().getUuid())){
1267                                 logger.warn("TaxonNameBase entity of existing taxon does not match Name in list -> replacing Name in list");
1268                                 nonViralName = HibernateProxyHelper.deproxy(taxon.getName(), NonViralName.class);
1269                         }
1270                 } else {
1271                         logger.info("creating new Taxon from TaxonName '" + nonViralName.getTitleCache()+"'");
1272                         taxon = Taxon.NewInstance(nonViralName, sec);
1273                 }
1274
1275                 //citation
1276                 Element elCitation = elScope.getChild("Citation",sddNamespace);
1277                 if (elCitation != null) {
1278                         String refCitation = elCitation.getAttributeValue("ref");
1279                         if (! refCitation.equals("")){
1280                                 citations.put(idCD, refCitation);
1281                         }
1282                         String location = elCitation.getAttributeValue("location");
1283                         if (! location.equals("")){
1284                                 locations.put(idCD, location);
1285                         }
1286                 }
1287                 return taxon;
1288         }
1289
1290         /**
1291          * @param sddNamespace
1292          * @param taxonDescription
1293          * @param elSummaryData
1294          */
1295         private void handleSummaryTextData(Namespace sddNamespace,
1296                         TaxonDescription taxonDescription, Element elSummaryData) {
1297                 String ref;
1298                 int k;
1299                 // <TextChar ref="c3">
1300                 List<Element> elTextChars = elSummaryData.getChildren("TextChar", sddNamespace);
1301                 k = 0;
1302                 //for each TextChar
1303                 for (Element elTextChar : elTextChars){
1304                         if ((++k % modCount) == 0){ logger.info("TextChar handled: " + (k-1));}
1305                         ref = elTextChar.getAttributeValue("ref");
1306                         Feature feature = features.get(ref);
1307                         TextData textData = TextData.NewInstance();
1308                         textData.setFeature(feature);
1309
1310                         // <Content>Free form text</Content>
1311                         String content = (String)ImportHelper.getXmlInputValue(elTextChar, "Content",sddNamespace);
1312                         textData.putText(datasetLanguage, content);
1313                         taxonDescription.addElement(textData);
1314                 }
1315         }
1316
1317         /**
1318          * @param sddNamespace
1319          * @param taxonDescription
1320          * @param elSummaryData
1321          */
1322         private void handleSummaryQuantitativeData(Namespace sddNamespace,
1323                         TaxonDescription taxonDescription, Element elSummaryData) {
1324                 String ref;
1325                 int k;
1326                 // <Quantitative ref="c2">
1327                 List<Element> elQuantitatives = elSummaryData.getChildren("Quantitative", sddNamespace);
1328                 k = 0;
1329                 //for each Quantitative
1330                 for (Element elQuantitative : elQuantitatives){
1331                         if ((++k % modCount) == 0){ logger.warn("Quantitative handled: " + (k-1));}
1332                         ref = elQuantitative.getAttributeValue("ref");
1333                         Feature feature = features.get(ref);
1334                         QuantitativeData quantitativeData = QuantitativeData.NewInstance();
1335                         quantitativeData.setFeature(feature);
1336
1337                         MeasurementUnit unit = units.get(ref);
1338                         String prefix = defaultUnitPrefixes.get(ref);
1339                         if (unit != null) {
1340                                 String u = unit.getLabel();
1341                                 if (prefix != null) {
1342                                         u = prefix + u;
1343                                 }
1344                                 unit.setLabel(u);
1345                                 quantitativeData.setUnit(unit);
1346                         }
1347
1348                         // <Measure type="Min" value="2.3"/>
1349                         List<Element> elMeasures = elQuantitative.getChildren("Measure", sddNamespace);
1350                         int l = 0;
1351
1352                         //for each State
1353                         for (Element elMeasure : elMeasures){
1354                                 if ((++l % modCount) == 0){ logger.info("States handled: " + (l-1));}
1355                                 String type = elMeasure.getAttributeValue("type");
1356                                 String value = elMeasure.getAttributeValue("value");
1357                                 if (value.contains(",")) {
1358                                         value = value.replace(',', '.');
1359                                 }
1360                                 Float v = Float.parseFloat(value);
1361                                 //Float v = new Float(0);
1362                                 StatisticalMeasure t = null;
1363                                 if (type.equals("Min")) {
1364                                         t = StatisticalMeasure.MIN();
1365                                 } else if (type.equals("Mean")) {
1366                                         t = StatisticalMeasure.AVERAGE();
1367                                 } else if (type.equals("Max")) {
1368                                         t = StatisticalMeasure.MAX();
1369                                 } else if (type.equals("SD")) {
1370                                         t = StatisticalMeasure.STANDARD_DEVIATION();
1371                                 } else if (type.equals("N")) {
1372                                         t = StatisticalMeasure.SAMPLE_SIZE();
1373                                 } else if (type.equals("UMethLower")) {
1374                                         t = StatisticalMeasure.TYPICAL_LOWER_BOUNDARY();
1375                                 } else if (type.equals("UMethUpper")) {
1376                                         t = StatisticalMeasure.TYPICAL_UPPER_BOUNDARY();
1377                                 } else if (type.equals("Var")) {
1378                                         t = StatisticalMeasure.VARIANCE();
1379                                 } else {
1380                                         t = StatisticalMeasure.NewInstance(type,type,type);
1381                                         statisticalMeasures.add(t);
1382                                 }
1383
1384                                 StatisticalMeasurementValue statisticalValue = StatisticalMeasurementValue.NewInstance();
1385                                 statisticalValue.setValue(v);
1386                                 statisticalValue.setType(t);
1387                                 quantitativeData.addStatisticalValue(statisticalValue);
1388                                 featureData.add(statisticalValue);
1389                         }
1390                         taxonDescription.addElement(quantitativeData);
1391                 }
1392         }
1393
1394         /**
1395          * @param sddNamespace
1396          * @param taxonDescription
1397          * @param elSummaryData
1398          */
1399         private void handleSummaryCategoricalData(Namespace sddNamespace,
1400                         TaxonDescription taxonDescription, Element elSummaryData) {
1401                 String ref;
1402                 // <Categorical ref="c4">
1403                 List<Element> elCategoricals = elSummaryData.getChildren("Categorical", sddNamespace);
1404                 int k = 0;
1405                 //for each Categorical
1406                 for (Element elCategorical : elCategoricals){
1407                         if ((++k % modCount) == 0){ logger.warn("Categorical handled: " + (k-1));}
1408                         ref = elCategorical.getAttributeValue("ref");
1409                         Feature feature = features.get(ref);
1410                         CategoricalData categoricalData = CategoricalData.NewInstance();
1411                         categoricalData.setFeature(feature);
1412
1413                         // <State ref="s3"/>
1414                         List<Element> elStates = elCategorical.getChildren("State", sddNamespace);
1415                         int l = 0;
1416
1417                         //for each State
1418                         for (Element elState : elStates){
1419                                 if ((++l % modCount) == 0){ logger.info("States handled: " + (l-1));}
1420                                 ref = elState.getAttributeValue("ref");
1421                                 State state = states.get(ref);
1422                                 if (state != null) {
1423                                         StateData stateData = StateData.NewInstance();
1424                                         stateData.setState(state);
1425                                         List<Element> elModifiers = elState.getChildren("Modifier", sddNamespace);
1426                                         for (Element elModifier : elModifiers){
1427                                                 ref = elModifier.getAttributeValue("ref");
1428                                                 Modifier modifier = modifiers.get(ref);
1429                                                 if (modifier != null) {
1430                                                         stateData.addModifier(modifier);
1431                                                 }
1432                                         }
1433                                         categoricalData.addState(stateData);
1434                                 }
1435                                 taxonDescription.addElement(categoricalData);
1436                         }
1437                 }
1438         }
1439
1440         // imports the persons associated with the dataset creation, modification, related publications
1441         protected void importAgents(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1442                 // <Agents>
1443                 logger.info("start Agents ...");
1444                 Element elAgents = elDataset.getChild("Agents",sddNamespace);
1445                 if (elAgents != null) {
1446                         // <Agent id="a1">
1447                         List <Element> listAgents = elAgents.getChildren("Agent", sddNamespace);
1448                         int j = 0;
1449                         //for each Agent
1450                         for (Element elAgent : listAgents){
1451
1452                                 try {
1453
1454                                         String idA = elAgent.getAttributeValue("id");
1455
1456                                         //  <Representation>
1457                                         //   <Label>Kevin Thiele</Label>
1458                                         //   <Detail role="Description">Ali Baba is also known as r.a.m.</Detail>
1459                                         //  </Representation>
1460                                         Person person = Person.NewInstance();
1461                                         importRepresentation(elAgent, sddNamespace, person, idA, cdmState);
1462                                         person.addSource(IdentifiableSource.NewDataImportInstance(idA, "Agent"));
1463
1464                                         /*XIM <Links>
1465                                         Element elLinks = elAgent.getChild("Links",sddNamespace);
1466
1467                                         if (elLinks != null) {
1468
1469                                                 //  <Link rel="Alternate" href="http://www.diversitycampus.net/people/hagedorn"/>
1470                                                 List<Element> listLinks = elLinks.getChildren("Link", sddNamespace);
1471                                                 int k = 0;
1472                                                 //for each Link
1473                                                 for (Element elLink : listLinks){
1474
1475                                                         try {
1476
1477                                                                 String rel = elLink.getAttributeValue("rel");
1478                                                                 String href = elLink.getAttributeValue("href");
1479
1480                                                                 Media link = Media.NewInstance();
1481                                                                 MediaRepresentation mr = MediaRepresentation.NewInstance();
1482                                                                 mr.addRepresentationPart(MediaRepresentationPart.NewInstance(href, null));
1483                                                                 link.addRepresentation(mr);
1484                                                                 person.addMedia(link);
1485
1486                                                         } catch (Exception e) {
1487                                                                 //FIXME
1488                                                                 logger.warn("Import of Link " + k + " failed.");
1489                                                                 success = false;
1490                                                         }
1491
1492                                                         if ((++k % modCount) == 0){ logger.info("Links handled: " + k);}
1493
1494                                                 }
1495                                         }
1496                                         */
1497                                         if (authors.containsKey(idA)) {
1498                                                 authors.put(idA,person);
1499                                         }
1500
1501                                         if (editors.containsKey(idA)) {
1502                                                 editors.put(idA, person);
1503                                         }
1504
1505                                 } catch (Exception e) {
1506                                         //FIXME
1507                                         logger.warn("Import of Agent " + j + " failed.");
1508                                         cdmState.setUnsuccessfull();
1509                                 }
1510
1511                                 if ((++j % modCount) == 0){ logger.info("Agents handled: " + j);}
1512
1513                         }
1514                 }
1515         }
1516
1517         // imports publications related with the data set
1518         protected void importPublications(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1519                 /* <Publications>
1520                           <Publication id="p112">
1521                             <Representation>
1522                               <Label>Gee, X. & Haa, Y. (2003). How to be happy in five minutes. Instant Gratifications, Palm Beach.</Label>
1523                             </Representation>
1524                             <Links>
1525                             <Link rel="BasedOn" href="doi:10.1992/32311"/>
1526                             <Link rel="Alternate" href="http://some.service.net/providing/bibliographic.data"/>
1527                             </Links>
1528                         </Publications>
1529 */
1530                 logger.info("start Publications ...");
1531                 Element elPublications = elDataset.getChild("Publications",sddNamespace);
1532
1533                 if (elPublications != null) {
1534                         List<Element> listPublications = elPublications.getChildren("Publication", sddNamespace);
1535                         int j = 0;
1536                         for (Element elPublication : listPublications){
1537
1538                                 try {
1539
1540                                         String idP = elPublication.getAttributeValue("id");
1541                                         Reference publication = ReferenceFactory.newArticle();
1542                                         importRepresentation(elPublication, sddNamespace, publication, idP, cdmState);
1543
1544                                         publications.put(idP,publication);
1545
1546                                 } catch (Exception e) {
1547                                         logger.warn("Import of Publication " + j + " failed.");
1548                                         cdmState.setUnsuccessfull();
1549                                 }
1550
1551                                 if ((++j % modCount) == 0){ logger.info("Publications handled: " + j);}
1552
1553                         }
1554                 }
1555         }
1556
1557         // imports media objects such as images //FIXME check mediaobj
1558         protected void importMediaObjects(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1559                 // <MediaObjects>
1560                 logger.info("start MediaObjects ...");
1561                 Element elMediaObjects = elDataset.getChild("MediaObjects",sddNamespace);
1562
1563                 if (elMediaObjects != null) {
1564                         // <MediaObject id="m1">
1565                         List<Element> listMediaObjects = elMediaObjects.getChildren("MediaObject", sddNamespace);
1566                         int j = 0;
1567                         for (Element elMO : listMediaObjects){
1568
1569                                 String id = "";
1570
1571                                 try {
1572                                         String idMO = elMO.getAttributeValue("id");
1573                                         id = idMO;
1574
1575                                         //  <Representation>
1576                                         //   <Label>Image description, e.g. to be used for alt-attribute in html.</Label>
1577                                         //  </Representation>
1578                                         Media media = Media.NewInstance();
1579                                         importRepresentation(elMO, sddNamespace, media, idMO, cdmState);
1580
1581                                         // <Type>Image</Type>
1582                                         // <Source href="http://test.edu/test.jpg"/>
1583                                         String type = (String)ImportHelper.getXmlInputValue(elMO,"Type",sddNamespace);
1584
1585                                         if ((type != null) && (type.equals("Image"))) {
1586                                                 Element elSource = elMO.getChild("Source",sddNamespace);
1587                                                 String href = elSource.getAttributeValue("href");
1588
1589                                                 ImageInfo imageMetaData = null;
1590                                                 ImageFile image = null;
1591                                                 if (href.substring(0,7).equals("http://")) {
1592                                                         try{
1593                                                                 URL url = new URL(href);
1594
1595                                                                 imageMetaData = ImageInfo.NewInstance(url.toURI(), 0);
1596                                                                 image = ImageFile.NewInstance(url.toURI(), null, imageMetaData);
1597                                                         } catch (MalformedURLException e) {
1598                                                                 logger.error("Malformed URL", e);
1599                                                         }
1600                                                 } else {
1601                                                         String sns = cdmState.getConfig().getSourceNameString();
1602                                                         File f = new File(sns);
1603                                                         File parent = f.getParentFile();
1604                                                         String fi = parent.toString() + File.separator + href;
1605                                                         File file = new File(fi);
1606                                                         imageMetaData = ImageInfo.NewInstance(new URI(fi), 0); //file
1607                                                         image = ImageFile.NewInstance(file.toURI(), null, imageMetaData);
1608                                                 }
1609                                                 MediaRepresentation representation = MediaRepresentation.NewInstance(imageMetaData.getMimeType(), null);
1610                                                 representation.addRepresentationPart(image);
1611
1612                                                 media.addRepresentation(representation);
1613
1614                                                 ArrayList<CdmBase> lcb = (ArrayList<CdmBase>) mediaObject_ListCdmBase.get(idMO);
1615                                                 if (lcb != null) {
1616                                                         for (int k = 0; k < lcb.size(); k++) {
1617                                                                 if (lcb.get(k) instanceof DefinedTermBase) {
1618                                                                         DefinedTermBase dtb = (DefinedTermBase) lcb.get(k);
1619                                                                         // if (lcb.get(0) instanceof DefinedTermBase) {
1620                                                                         // DefinedTermBase dtb = (DefinedTermBase) lcb.get(0);
1621                                                                         //                                                                      if (dtb!=null) {
1622                                                                         //                                                                              if (k == 0) {
1623                                                                         dtb.addMedia(media);
1624                                                                         //System.out.println(dtb.getLabel());
1625                                                                         //                                                                              } else {
1626                                                                         //                                                                                      Media me = (Media) media.clone();
1627                                                                         //                                                                                      dtb.addMedia(me);
1628                                                                         //                                                                              }
1629                                                                         //                                                                      }
1630                                                                 } else if (lcb.get(k) instanceof Reference) {
1631                                                                         Reference rb = (Reference) lcb.get(k);
1632                                                                         //} else if (lcb.get(0) instanceof Reference) {
1633                                                                         //Reference rb = (Reference) lcb.get(0);
1634                                                                         // rb.setTitleCache(label);
1635                                                                         //                                                                      if (rb!=null) {
1636                                                                         //                                                                              if (k == 0) {
1637                                                                         rb.addMedia(media);
1638                                                                         //System.out.println(rb.getTitle());
1639                                                                         //                                                                              } else {
1640                                                                         //                                                                                      Media me = (Media) media.clone();
1641                                                                         //                                                                                      rb.addMedia(me);
1642                                                                         //                                                                              }
1643                                                                         //                                                                      }
1644                                                                 }/* else if (lcb.get(k) instanceof TaxonNameBase){
1645                                                                         TaxonNameBase tb = (TaxonNameBase) lcb.get(k);
1646                                                                         tb.addMedia(media);
1647                                                         }*/
1648                                                         }
1649                                                 }
1650                                         }
1651
1652                                 } catch (Exception e) {
1653                                         //FIXME
1654                                         logger.warn("Could not attach MediaObject " + j + "(SDD: " + id + ") to several objects.");
1655                                         cdmState.setUnsuccessfull();
1656                                 }
1657
1658                                 if ((++j % modCount) == 0){ logger.info("MediaObjects handled: " + j);
1659
1660                                 }
1661                         }
1662                 }
1663         }
1664
1665         // imports the <DescriptiveConcepts> block ; DescriptiveConcepts are used as nodes in CharacterTrees and Characters as leaves
1666         // but since Modifiers can be linked to DescriptiveConcepts they are stored as features with a particular Marker
1667         protected void importDescriptiveConcepts(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1668                 /* <DescriptiveConcepts>
1669                       <DescriptiveConcept id="dc0">
1670                                 <Representation>
1671                                   <Label>Fixed set of modifiers supported in Lucid3</Label>
1672                                 </Representation>
1673                                 <Modifiers>
1674                                   <Modifier id="mod1">
1675                                     <Representation>
1676                                       <Label>rarely</Label>
1677                                     </Representation>
1678                                     <ModifierClass>Frequency</ModifierClass>
1679                                     <ProportionRange lowerestimate="0.0" upperestimate="0.25"/>
1680                                   </Modifier>
1681                           </Modifiers>
1682                         </DescriptiveConcept>
1683                  </DescriptiveConcepts>
1684                  */
1685                 logger.info("start DescriptiveConcepts ...");
1686                 Element elDescriptiveConcepts = elDataset.getChild("DescriptiveConcepts",sddNamespace);
1687                 if (elDescriptiveConcepts != null) {
1688                         List<Element> listDescriptiveConcepts = elDescriptiveConcepts.getChildren("DescriptiveConcept", sddNamespace);
1689                         int j = 0;
1690
1691                         for (Element elDescriptiveConcept : listDescriptiveConcepts){
1692                                 try {
1693                                 String id = elDescriptiveConcept.getAttributeValue("id");
1694                                         Feature feature = Feature.NewInstance();
1695                                         feature.setKindOf(Feature.DESCRIPTION());
1696                                         if (!id.equals("")) {
1697                                         //       <Representation>
1698                                         //       <Label>Body</Label>
1699                                         importRepresentation(elDescriptiveConcept, sddNamespace, feature, id, cdmState);
1700                                                 features.put(id, feature);
1701                                                 getTermService().save(feature);//XIM
1702                                                 descriptiveConcepts.add(feature);
1703                                                 // imports the modifiers
1704                                                 Element elModifiers = elDescriptiveConcept.getChild("Modifiers", sddNamespace);
1705                                         if (elModifiers !=null){
1706                                                 List<Element> listModifiers = elModifiers.getChildren("Modifier", sddNamespace);
1707                                                         TermVocabulary<Modifier> termVocabularyState = TermVocabulary.NewInstance(null, null, null, null);
1708                                                 for (Element elModifier : listModifiers) {
1709                                                                 Modifier modif = Modifier.NewInstance();
1710                                                                 String idmod = elModifier.getAttributeValue("id");
1711                                                                 importRepresentation(elModifier, sddNamespace, modif, idmod, cdmState);
1712                                                                 termVocabularyState.addTerm(modif);
1713                                                                 //termVocabularyStates.add(termVocabularyState);
1714                                                                 getVocabularyService().save(termVocabularyState);//XIM
1715                                                                 modifiers.put(idmod, modif);
1716                                                 }
1717                                                         feature.addRecommendedModifierEnumeration(termVocabularyState);
1718                                 }
1719
1720                                         }
1721                                 }
1722                                 catch (Exception e) {
1723                                         logger.warn("Import of DescriptiveConcept " + j + " failed: " + e.getMessage());
1724                                 }
1725                                 if ((++j % modCount) == 0){ logger.info("DescriptiveConcepts handled: " + j);}
1726
1727                         }
1728                 }
1729         }
1730
1731         // imports the <CharacterTrees> block
1732         protected void importCharacterTrees(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1733                 // <CharacterTrees>
1734                 logger.info("start CharacterTrees ...");
1735                 Element elCharacterTrees = elDataset.getChild("CharacterTrees",sddNamespace);
1736
1737                 if (elCharacterTrees != null) {
1738                         List<Element> listCharacterTrees = elCharacterTrees.getChildren("CharacterTree", sddNamespace);
1739                         int j = 0;
1740                         for (Element elCharacterTree : listCharacterTrees){
1741                                 try {
1742                                         Element elRepresentation = elCharacterTree.getChild("Representation",sddNamespace);
1743                                         String label = (String)ImportHelper.getXmlInputValue(elRepresentation,"Label",sddNamespace);
1744                                         //Element elDesignedFor = elCharacterTree.getChild("DesignedFor",sddNamespace);//TODO ?
1745
1746                                         FeatureTree featureTree =  FeatureTree.NewInstance();
1747                                         importRepresentation(elCharacterTree, sddNamespace, featureTree, "", cdmState);
1748                                         FeatureNode root = featureTree.getRoot();
1749                                         List<Element> listeOfNodes = elCharacterTree.getChildren("Nodes", sddNamespace);
1750
1751                                         //Nodes of CharacterTrees in SDD always refer to DescriptiveConcepts
1752                                         for (Element elNodes : listeOfNodes) {
1753                                                 handleCharacterNodes(sddNamespace, root, elNodes);
1754                                         }
1755                                         featureTrees.add(featureTree);
1756                                         if (workingSet.getDescriptiveSystem() != null){
1757                                                 //TODO how to handle multiple
1758                                                 logger.warn("Multiple feature trees not yet supported");
1759                                         }else{
1760                                                 workingSet.setDescriptiveSystem(featureTree);
1761                                         }
1762                                 }
1763
1764                                 catch (Exception e) {
1765                                         logger.warn("Import of Character tree " + j + " failed.");
1766                                         cdmState.setUnsuccessfull();
1767                                 }
1768                                 if ((++j % modCount) == 0){ logger.info("CharacterTrees handled: " + j);}
1769
1770                         }
1771
1772                 }
1773         }
1774
1775         /**
1776          * @param sddNamespace
1777          * @param root
1778          * @param elNodes
1779          */
1780         private void handleCharacterNodes(Namespace sddNamespace, FeatureNode root, Element elNodes) {
1781                 List<Element> listNodes = elNodes.getChildren("Node", sddNamespace);
1782                 if (listNodes != null) {
1783                         for (Element elNode : listNodes){
1784                                 String idN = elNode.getAttributeValue("id");
1785                                 FeatureNode fn = null;
1786                                 Feature dc = null;
1787                                 if (idN!=null) {
1788                                         // DescriptiveConcepts are used as nodes in CharacterTrees
1789                                         Element elDescriptiveConcept = elNode.getChild("DescriptiveConcept", sddNamespace);
1790                                         if (elDescriptiveConcept != null){
1791                                                 String refDC = elDescriptiveConcept.getAttributeValue("ref");
1792                                                 dc = features.get(refDC);
1793                                                 fn = FeatureNode.NewInstance(dc);
1794                                         }
1795                                         if (fn==null){
1796                                                 fn = FeatureNode.NewInstance();
1797                                         }
1798                                         Element elParent = elNode.getChild("Parent", sddNamespace);
1799                                         // in SDD links between Nodes are referenced by the <Parent> tag
1800                                         if (elParent!=null){
1801                                                 String refP = elParent.getAttributeValue("ref");
1802                                                 if (refP!=null) {
1803                                                         FeatureNode parent = featureNodes.get(refP);
1804                                                         if (parent==null){
1805                                                                 root.addChild(fn); // if no parent found or the reference is broken, add the node to the root of the tree
1806                                                         }
1807                                                         else {
1808                                                                 parent.addChild(fn);
1809                                                         }
1810                                                 }
1811                                         }
1812                                         else {
1813                                                 root.addChild(fn); // if no parent found or the reference is broken, add the node to the root of the tree
1814                                         }
1815                                 }
1816                                 featureNodes.put(idN, fn);
1817                         }
1818                 }
1819
1820                 // Leaves of CharacterTrees in SDD are always CharNodes (referring to Characters)
1821                 List<Element> listCharNodes = elNodes.getChildren("CharNode", sddNamespace);
1822                 if (listCharNodes != null) {
1823                         for (Element elCharNode : listCharNodes){
1824                                 Element elParent = elCharNode.getChild("Parent", sddNamespace);
1825                                 Element elCharacter = elCharNode.getChild("Character", sddNamespace);
1826                                 Element elDependencyRules = elCharNode.getChild("DependencyRules", sddNamespace);
1827                                 FeatureNode fn = FeatureNode.NewInstance();
1828
1829                                 if (elDependencyRules!=null){
1830                                         Element elInapplicableIf = elCharNode.getChild("InapplicableIf", sddNamespace);
1831                                         if (elInapplicableIf!=null){
1832                                                 List<Element> listStates = elInapplicableIf.getChildren("State", sddNamespace);
1833                                                 for (Element stateElement : listStates) {
1834                                                         String refState = stateElement.getAttributeValue("ref");
1835                                                         if ((refState!=null)&&(!refState.equals(""))) {
1836                                                                 State state = states.get(refState);
1837                                                                 fn.addInapplicableState(state);
1838                                                         }
1839                                                 }
1840                                         }
1841                                         Element elOnlyapplicableIf = elCharNode.getChild("OnlyApplicableIf", sddNamespace);
1842                                         if (elOnlyapplicableIf!=null){
1843                                                 List<Element> listStates = elInapplicableIf.getChildren("State", sddNamespace);
1844                                                 for (Element stateElement : listStates) {
1845                                                         String refState = stateElement.getAttributeValue("ref");
1846                                                         if ((refState!=null)&&(!refState.equals(""))) {
1847                                                                 State state = states.get(refState);
1848                                                                 fn.addApplicableState(state);
1849                                                         }
1850                                                 }
1851                                         }
1852                                 }
1853
1854                                 if (elParent!=null){
1855                                         String refP = elParent.getAttributeValue("ref");
1856                                         if ((refP!=null)&&(!refP.equals(""))) {
1857                                         FeatureNode parent = featureNodes.get(refP);
1858                                                 if (parent==null){
1859                                                 parent = root; // if no parent found or the reference is broken, add the node to the root of the tree
1860                                                 }
1861                                                 parent.addChild(fn);
1862                                         }
1863                                 }
1864                                 String refC = elCharacter.getAttributeValue("ref");
1865                                 if ((refC!=null)&&(!refC.equals(""))){
1866                                         Feature character = features.get(refC);
1867                                         fn.setFeature(character);
1868                                         featureNodes.put(refC, fn);
1869                                 }
1870                         }
1871                 }
1872         }
1873
1874         // imports the <TaxonHierarchies> block
1875         protected void importTaxonHierarchies(Element elDataset, Namespace sddNamespace, SDDImportState cdmState){
1876
1877                 logger.info("start TaxonHierarchies ...");
1878                 Element elTaxonHierarchies = elDataset.getChild("TaxonHierarchies",sddNamespace);
1879
1880                 if (elTaxonHierarchies != null) {
1881                         List<Element> listTaxonHierarchies = elTaxonHierarchies.getChildren("TaxonHierarchy", sddNamespace);
1882                         int j = 0;
1883                         for (Element elTaxonHierarchy : listTaxonHierarchies){
1884                                 try {
1885                                         Element elRepresentation = elTaxonHierarchy.getChild("Representation",sddNamespace);
1886                                         String label = (String)ImportHelper.getXmlInputValue(elRepresentation,"Label",sddNamespace);
1887                                                 Classification classification =  Classification.NewInstance(label);
1888                                                 importRepresentation(elTaxonHierarchy, sddNamespace, classification, "", cdmState);
1889
1890                                                 Set<TaxonNode> root = classification.getChildNodes();
1891                                                 Element elNodes = elTaxonHierarchy.getChild("Nodes", sddNamespace); // There can be only one <Nodes> block for TaxonHierarchies
1892                                                 List<Element> listNodes = elNodes.getChildren("Node", sddNamespace);
1893
1894                                                 for (Element elNode : listNodes){
1895                                                         String idN = elNode.getAttributeValue("id");
1896                                                         TaxonNameBase tnb = null;
1897                                                         if (!idN.equals("")) {
1898                                                                 Element elTaxonName = elNode.getChild("TaxonName", sddNamespace);
1899                                                                 String refTN = elTaxonName.getAttributeValue("ref");
1900                                                                 tnb = taxonNameBases.get(refTN);
1901                                                                 Taxon taxon = (Taxon) tnb.getTaxa().iterator().next() ;
1902                                                                 Element elParent = elNode.getChild("Parent", sddNamespace);
1903                                                                 if (elParent!=null){
1904                                                                         String refP = elParent.getAttributeValue("ref");
1905                                                                         if (!refP.equals("")) {
1906                                                                                 TaxonNode parent = taxonNodes.get(refP);
1907                                                                                 TaxonNode child = parent.addChildTaxon(taxon, sec, "", Synonym.NewInstance(tnb, sec));
1908                                                                                 taxonNodes.put(idN,child);
1909                                                                         }
1910                                                                 }
1911                                                                 else {
1912                                                                         TaxonNode tn = classification.addChildTaxon(taxon, sec, "", Synonym.NewInstance(tnb, sec)); // if no parent found or the reference is broken, add the node to the root of the tree
1913                                                                         taxonNodes.put(idN,tn);
1914                                                                 }
1915                                                         }
1916                                                 }
1917
1918                                                 classifications.add(classification);
1919                                         }
1920
1921                                 catch (Exception e) {
1922                                         //FIXME
1923                                         logger.warn("Import of Taxon Hierarchy " + j + " failed.");
1924                                         cdmState.setUnsuccessfull();
1925                                 }
1926
1927                                 if ((++j % modCount) == 0){ logger.info("TaxonHierarchies handled: " + j);}
1928
1929                         }
1930
1931                 }
1932         }
1933
1934
1935         // imports the <GeographicAreas> block
1936         protected void importGeographicAreas(Element elDataset, Namespace sddNamespace, SDDImportState cdmState) {
1937                 Element elGeographicAreas = elDataset.getChild("GeographicAreas",sddNamespace);
1938                 if (elGeographicAreas != null) {
1939                         List<Element> listGeographicAreas = elGeographicAreas.getChildren("GeographicArea", sddNamespace);
1940                         int j = 0;
1941
1942                         for (Element elGeographicArea : listGeographicAreas){
1943
1944                                 String id = elGeographicArea.getAttributeValue("id");
1945                                 NamedArea na = new NamedArea();
1946                                 importRepresentation(elGeographicArea, sddNamespace, na, id, cdmState);
1947                                 namedAreas.put(id,na);
1948                                                                 }
1949                         if ((++j % modCount) == 0){ logger.info("GeographicAreas handled: " + j);}
1950                                                         }
1951                                                         }
1952 }