cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/sdd/in/SDDImport.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.sdd.in;
  11
  12 import java.io.File;
  13 import java.net.MalformedURLException;
  14 import java.net.URI;
  15 import java.net.URL;
  16 import java.text.SimpleDateFormat;
  17 import java.util.ArrayList;
  18 import java.util.Date;
  19 import java.util.HashMap;
  20 import java.util.HashSet;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Set;
  24
  25 import org.apache.commons.lang.StringUtils;
  26 import org.apache.log4j.Logger;
  27 import org.jdom.Element;
  28 import org.jdom.Namespace;
  29 import org.joda.time.DateTime;
  30 import org.springframework.stereotype.Component;
  31 import org.springframework.transaction.TransactionStatus;
  32
  33 import eu.etaxonomy.cdm.api.service.IDescriptionService;
  34 import eu.etaxonomy.cdm.api.service.IService;
  35 import eu.etaxonomy.cdm.common.mediaMetaData.ImageMetaData;
  36 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  37 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  38 import eu.etaxonomy.cdm.io.common.ICdmImport;
  39 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  40 import eu.etaxonomy.cdm.io.common.ImportHelper;
  41 import eu.etaxonomy.cdm.io.sdd.SDDTransformer;
  42 import eu.etaxonomy.cdm.model.agent.Person;
  43 import eu.etaxonomy.cdm.model.agent.Team;
  44 import eu.etaxonomy.cdm.model.common.Annotation;
  45 import eu.etaxonomy.cdm.model.common.AnnotationType;
  46 import eu.etaxonomy.cdm.model.common.CdmBase;
  47 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  48 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  49 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  50 import eu.etaxonomy.cdm.model.common.Language;
  51 import eu.etaxonomy.cdm.model.common.LanguageString;
  52 import eu.etaxonomy.cdm.model.common.Marker;
  53 import eu.etaxonomy.cdm.model.common.MarkerType;
  54 import eu.etaxonomy.cdm.model.common.Representation;
  55 import eu.etaxonomy.cdm.model.common.TermBase;
  56 import eu.etaxonomy.cdm.model.common.TermVocabulary;
  57 import eu.etaxonomy.cdm.model.common.VersionableEntity;
  58 import eu.etaxonomy.cdm.model.description.CategoricalData;
  59 import eu.etaxonomy.cdm.model.description.Feature;
  60 import eu.etaxonomy.cdm.model.description.FeatureNode;
  61 import eu.etaxonomy.cdm.model.description.FeatureTree;
  62 import eu.etaxonomy.cdm.model.description.MeasurementUnit;
  63 import eu.etaxonomy.cdm.model.description.Modifier;
  64 import eu.etaxonomy.cdm.model.description.QuantitativeData;
  65 import eu.etaxonomy.cdm.model.description.State;
  66 import eu.etaxonomy.cdm.model.description.StateData;
  67 import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
  68 import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
  69 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  70 import eu.etaxonomy.cdm.model.description.TextData;
  71 import eu.etaxonomy.cdm.model.location.NamedArea;
  72 import eu.etaxonomy.cdm.model.media.IdentifiableMediaEntity;
  73 import eu.etaxonomy.cdm.model.media.ImageFile;
  74 import eu.etaxonomy.cdm.model.media.Media;
  75 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
  76 import eu.etaxonomy.cdm.model.media.MediaRepresentationPart;
  77 import eu.etaxonomy.cdm.model.media.Rights;
  78 import eu.etaxonomy.cdm.model.name.NonViralName;
  79 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  80 import eu.etaxonomy.cdm.model.occurrence.Specimen;
  81 import eu.etaxonomy.cdm.model.reference.ReferenceBase;
  82 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  83 import eu.etaxonomy.cdm.model.taxon.Synonym;
  84 import eu.etaxonomy.cdm.model.taxon.Taxon;
  85 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  86 import eu.etaxonomy.cdm.model.taxon.TaxonomicTree;
  87
  88 /**
  89  * @author h.fradin
  90  * @created 24.10.2008
  91  * @version 1.0
  92  */
  93 @Component("sddImport")
  94 public class SDDImport extends CdmImportBase<SDDImportConfigurator, SDDImportState> implements ICdmImport<SDDImportConfigurator, SDDImportState> {
  95         private static final Logger logger = Logger.getLogger(SDDImport.class);
  96
  97         private static int modCount = 1000;
  98
  99         private Map<String,Person> authors = new HashMap<String,Person>();
 100         private Map<String,String> citations = new HashMap<String,String>();
 101         private Map<String,String> defaultUnitPrefixes = new HashMap<String,String>();
 102         private Map<String,Person> editors = new HashMap<String,Person>();
 103         private Map<String,FeatureNode> featureNodes = new HashMap<String,FeatureNode>();
 104         private Map<String,Feature> features = new HashMap<String,Feature>();
 105         private Map<String,String> locations = new HashMap<String,String>();
 106         private Map<String,List<CdmBase>> mediaObject_ListCdmBase = new HashMap<String,List<CdmBase>>();
 107         private Map<String,String> mediaObject_Role = new HashMap<String,String>();
 108         private Map<String,ReferenceBase> publications = new HashMap<String,ReferenceBase>();
 109         private Map<String,State> states = new HashMap<String,State>();
 110         private Map<String,TaxonDescription> taxonDescriptions = new HashMap<String,TaxonDescription>();
 111         private Map<String,NonViralName> taxonNameBases = new HashMap<String,NonViralName>();
 112         private Map<String,MeasurementUnit> units = new HashMap<String,MeasurementUnit>();
 113         private Map<String,TaxonNode> taxonNodes = new HashMap<String,TaxonNode>();
 114         private Map<String,NamedArea> namedAreas = new HashMap<String,NamedArea>();
 115         private Map<String,Specimen> specimens = new HashMap<String,Specimen>();
 116         private Map<String,Modifier> modifiers = new HashMap<String,Modifier>();
 117
 118         private Set<MarkerType> markerTypes = new HashSet<MarkerType>();
 119         private Set<TermVocabulary> vocabularies = new HashSet<TermVocabulary>();
 120
 121         private Set<Feature> descriptiveConcepts = new HashSet<Feature>();
 122         private Set<AnnotationType> annotationTypes = new HashSet<AnnotationType>();
 123         private Set<Feature> featureSet = new HashSet<Feature>();
 124         private Set<ReferenceBase> sources = new HashSet<ReferenceBase>();
 125         private ReferenceBase sec = ReferenceFactory.newDatabase();
 126         private ReferenceBase sourceReference = null;
 127
 128         private Language datasetLanguage = null;
 129
 130         private Namespace xmlNamespace = Namespace.getNamespace("xml","http://www.w3.org/XML/1998/namespace");
 131
 132         private String generatorName = "";
 133         private String generatorVersion = "";
 134
 135         private Set<StatisticalMeasure> statisticalMeasures = new HashSet<StatisticalMeasure>();
 136         private Set<VersionableEntity> featureData = new HashSet<VersionableEntity>();
 137         private Set<FeatureTree> featureTrees = new HashSet<FeatureTree>();
 138         private Set<TaxonomicTree> taxonomicTrees = new HashSet<TaxonomicTree>();
 139
 140         private Rights copyright = null;
 141
 142         private int taxonNamesCount = 0; //XIM ajout
 143
 144         public SDDImport(){
 145                 super();
 146         }
 147
 148         @Override
 149         public boolean doCheck(SDDImportState state){
 150                 boolean result = true;
 151                 logger.warn("No check implemented for SDD");
 152                 return result;
 153         }
 154
 155         //      @Override
 156         //      public boolean doInvoke(IImportConfigurator config, Map<String, MapWrapper<? extends CdmBase>> stores){
 157         @Override
 158         public boolean doInvoke(SDDImportState state){
 159                 boolean success = true;
 160
 161                 TransactionStatus ts = startTransaction();
 162                 SDDImportConfigurator sddConfig = state.getConfig();
 163
 164                 logger.info("start Datasets ...");
 165                 // <Datasets xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://rs.tdwg.org/UBIF/2006/" xsi:schemaLocation="http://rs.tdwg.org/UBIF/2006/ ../SDD.xsd">
 166                 Element root = sddConfig.getSourceRoot();
 167                 Namespace sddNamespace = sddConfig.getSddNamespace();
 168
 169                 logger.info("start TechnicalMetadata ...");
 170                 // <TechnicalMetadata created="2006-04-20T10:00:00">
 171                 importTechnicalMetadata(root, sddNamespace, sddConfig);
 172                 List<Element> elDatasets = root.getChildren("Dataset",sddNamespace);
 173                 int i = 0;
 174
 175                 //for each Dataset
 176                 logger.info("start Dataset ...");
 177                 for (Element elDataset : elDatasets){
 178                         success &= importDataset(elDataset, sddNamespace, state);
 179                         if ((++i % modCount) == 0){ logger.info("dataset(s) handled: " + i);}
 180                         logger.info(i + " dataset(s) handled");
 181                 }
 182                 commitTransaction(ts);
 183                 logger.info("End of transaction");
 184                 return success;
 185         }
 186
 187         /* (non-Javadoc)
 188          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 189          */
 190         protected boolean isIgnore(SDDImportState state){
 191                 return false;
 192         }
 193
 194
 195         // associates the reference of a media object in SDD with a CdmBase Object
 196         protected void associateImageWithCdmBase(String refMO, CdmBase cb){
 197                 if ((refMO != null) && (cb!=null)) {
 198                         if (! refMO.equals("")) {
 199                                 if (! mediaObject_ListCdmBase.containsKey(refMO)) {
 200                                         List<CdmBase> lcb = new ArrayList<CdmBase>();
 201                                         lcb.add(cb);
 202                                         mediaObject_ListCdmBase.put(refMO,lcb);
 203                                 } else {
 204                                         List<CdmBase> lcb = mediaObject_ListCdmBase.get(refMO);
 205                                         lcb.add(cb);
 206                                         mediaObject_ListCdmBase.put(refMO,lcb);
 207                                 }
 208                         }
 209                 }
 210         }
 211
 212         // imports information about the Dataset
 213         protected void importDatasetRepresentation(Element parent, Namespace sddNamespace){
 214                 logger.info("start Representation ...");
 215                 /* <Representation>
 216                         <Label>The Genus Viola</Label>
 217                         <Detail>This is an example for a very simple SDD file, representing a single description with categorical, quantitative, and text character. Compare also the "Fragment*" examples, which contain more complex examples in the form of document fragments. Intended for version="SDD 1.1".</Detail>
 218                </Representation>
 219                  */
 220                 Element elRepresentation = parent.getChild("Representation",sddNamespace);
 221                 String label = (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace);
 222                 String detail = (String)ImportHelper.getXmlInputValue(elRepresentation, "Detail",sddNamespace);
 223
 224                 sec.setTitleCache(label, true);
 225
 226                 if (detail != null) {
 227                         Annotation annotation = Annotation.NewInstance(detail, datasetLanguage);
 228                         annotation.setAnnotationType(AnnotationType.EDITORIAL());
 229                         sec.addAnnotation(annotation);
 230                 }
 231
 232                 List<Element> listMediaObjects = elRepresentation.getChildren("MediaObject",sddNamespace);
 233
 234                 for (Element elMediaObject : listMediaObjects) {
 235                         String ref = null;
 236                         String role = null;
 237                         if (elMediaObject != null) {
 238                                 ref = elMediaObject.getAttributeValue("ref");
 239                                 role = elMediaObject.getAttributeValue("role");
 240                         }
 241                         if (ref != null) {
 242                                 if (!ref.equals("")) {
 243                                         this.associateImageWithCdmBase(ref,sourceReference);
 244                                         this.associateImageWithCdmBase(ref,sec);
 245                                         mediaObject_Role.put(ref,role);
 246                                 }
 247                         }
 248                 }
 249         }
 250
 251         // imports the representation (label, detail, lang) of a particular SDD element
 252         protected void importRepresentation(Element parent, Namespace sddNamespace, VersionableEntity ve, String id, IImportConfigurator config){
 253                 Element elRepresentation = parent.getChild("Representation",sddNamespace);
 254
 255                 Map<Language,List<String>> langLabDet = new HashMap<Language,List<String>>();
 256
 257                 handleRepresentationLabels(sddNamespace, elRepresentation, langLabDet);
 258                 handleRepresentationDetails(sddNamespace, elRepresentation, langLabDet);
 259
 260                 if (ve instanceof TermBase) {
 261                         makeRepresentationForTerms((TermBase)ve, langLabDet);
 262                 }else if (ve instanceof Media) {
 263                         makeRepresentationForMedia((Media)ve, langLabDet);
 264                 }else if (ve instanceof IdentifiableEntity<?>) {
 265                         IdentifiableEntity<?> ie = (IdentifiableEntity<?>)ve;
 266                         makeRepresentationForIdentifiableEntity(sddNamespace, ie, elRepresentation, langLabDet);
 267                         if (ve instanceof IdentifiableMediaEntity<?>){
 268                                 makeRepresentationForIdentifiableMediaEntity(parent, sddNamespace, (IdentifiableMediaEntity<?>)ve);
 269                         }
 270                 }
 271
 272                 makeRepresentationMediaObjects(sddNamespace, ve, elRepresentation);//FIXME
 273
 274         }
 275
 276
 277         /**
 278          * Handles the "Detail" children of representations. Adds the result to the langLabDet.
 279          * @param sddNamespace
 280          * @param elRepresentation
 281          * @param langLabDet
 282          */
 283         private void handleRepresentationDetails(Namespace sddNamespace,
 284                         Element elRepresentation, Map<Language, List<String>> langLabDet) {
 285                 List<Element> listDetails = elRepresentation.getChildren("Detail",sddNamespace);
 286                 for (Element elDetail : listDetails){
 287                         Language language = getLanguage(elDetail);
 288                         String role = elDetail.getAttributeValue("role");
 289                         String detail = elDetail.getText();
 290                         List<String> labDet = langLabDet.get(language);
 291                         labDet.add(detail);
 292                         labDet.add(role);
 293                         langLabDet.put(language, labDet);
 294                 }
 295         }
 296
 297         /**
 298          * Handles the "Label" children of representations. Adds the result to the langLabDet.
 299          * @param sddNamespace
 300          * @param elRepresentation
 301          * @param langLabDet
 302          */
 303         private void handleRepresentationLabels(Namespace sddNamespace,
 304                                 Element elRepresentation, Map<Language, List<String>> langLabDet) {
 305                 // <Label xml:lang="la">Viola hederacea Labill.</Label>
 306                 List<Element> listLabels = elRepresentation.getChildren("Label",sddNamespace);
 307                 for (Element elLabel : listLabels){
 308                         Language language = getLanguage(elLabel);
 309                         String label = elLabel.getText();
 310                         List<String> labDet = new ArrayList<String>(3);
 311                         labDet.add(label);
 312                         langLabDet.put(language, labDet);
 313                 }
 314         }
 315
 316         /**
 317          *
 318          * @param ve
 319          * @param langLabDet
 320          */
 321         private void makeRepresentationForMedia(Media m, Map<Language, List<String>> langLabDet) {
 322                 for (Language lang : langLabDet.keySet()){
 323                         List<String> labDet = langLabDet.get(lang);
 324                         if (labDet.get(0) != null){
 325                                 m.addTitle(LanguageString.NewInstance(labDet.get(0), lang));
 326                         }
 327                         if (labDet.size()>1) {
 328                                 m.addDescription(labDet.get(1), lang);
 329                         }
 330                 }
 331         }
 332
 333         /**
 334          * Handles representations for terms. Adds one representation per language in langLabDet.
 335          *
 336          * @param ve
 337          * @param langLabDet
 338          */
 339         private void makeRepresentationForTerms(TermBase tb, Map<Language, List<String>> langLabDet) {
 340                         for (Language lang : langLabDet.keySet()){
 341                                 List<String> labDet = langLabDet.get(lang);
 342                                 if (labDet.size()>0){
 343                                         if (labDet.size()>1) {
 344                                                 tb.addRepresentation(Representation.NewInstance(labDet.get(1), labDet.get(0), labDet.get(0), lang));
 345                                         } else {
 346                                                 tb.addRepresentation(Representation.NewInstance(labDet.get(0), labDet.get(0), labDet.get(0), lang));
 347                                         }
 348                                 }
 349                         }
 350         }
 351
 352
 353         /**
 354          * Handles the "MediaObject" children of representations.
 355          * @param sddNamespace
 356          * @param ve
 357          * @param elRepresentation
 358          */
 359         private void makeRepresentationMediaObjects(Namespace sddNamespace,     VersionableEntity ve, Element elRepresentation) {
 360                 List <Element> listMediaObjects = elRepresentation.getChildren("MediaObject", sddNamespace);
 361                 for (Element elMediaObject : listMediaObjects) {
 362                         String ref = null;
 363                         //TODO
 364                         String role = null;
 365                         if (elMediaObject != null) {
 366                                 ref = elMediaObject.getAttributeValue("ref");
 367                                 role = elMediaObject.getAttributeValue("role");
 368                         }
 369                         if (StringUtils.isNotBlank(ref)) {
 370                                 if (ve instanceof TaxonDescription) {
 371                                         TaxonDescription td = (TaxonDescription) ve;
 372                                         if (td.getDescriptionSources().size() > 0) {
 373                                                 this.associateImageWithCdmBase(ref,(ReferenceBase) td.getDescriptionSources().toArray()[0]);
 374                                         } else {
 375                                                 ReferenceBase descriptionSource = ReferenceFactory.newGeneric();
 376                                                 sources.add(descriptionSource);
 377                                                 td.addDescriptionSource(descriptionSource);
 378                                                 this.associateImageWithCdmBase(ref,descriptionSource);
 379                                         }
 380                                 } else {
 381                                         this.associateImageWithCdmBase(ref,ve);
 382                                 }
 383                         }
 384                 }
 385         }
 386
 387         /**
 388          * Handles the "Links" element
 389          * @param parent
 390          * @param sddNamespace
 391          * @param ve
 392          */
 393         private void makeRepresentationForIdentifiableMediaEntity(Element parent,
 394                         Namespace sddNamespace, IdentifiableMediaEntity ime) {
 395                 Element elLinks = parent.getChild("Links",sddNamespace);
 396
 397                 if (elLinks != null) {
 398
 399                         //  <Link rel="Alternate" href="http://www.diversitycampus.net/people/hagedorn"/>
 400                         List<Element> listLinks = elLinks.getChildren("Link", sddNamespace);
 401                         Media link = Media.NewInstance();
 402                         MediaRepresentation mr = MediaRepresentation.NewInstance();
 403                         int k = 0;
 404                         //for each Link
 405                         for (Element elLink : listLinks){
 406
 407                                 try {
 408                                         //TODO
 409                                         String rel = elLink.getAttributeValue("rel");
 410                                         String href = elLink.getAttributeValue("href");
 411
 412                                         mr.addRepresentationPart(MediaRepresentationPart.NewInstance(href, null));
 413                                         link.addRepresentation(mr);
 414                                         ime.addMedia(link);
 415
 416                                 } catch (Exception e) {
 417                                         //FIXME
 418                                         logger.warn("Import of Link " + k + " failed.");
 419                                 }
 420
 421                                 if ((++k % modCount) == 0){ logger.info("Links handled: " + k);}
 422
 423                         }
 424                 }
 425         }
 426
 427         /**
 428          * @param sddNamespace
 429          * @param ve
 430          * @param elRepresentation
 431          * @param langLabDet
 432          * @return
 433          */
 434         private void makeRepresentationForIdentifiableEntity(Namespace sddNamespace, IdentifiableEntity<?> ie,
 435                                         Element elRepresentation, Map<Language, List<String>> langLabDet) {
 436                 List<String> labDet = null;
 437
 438                 if (ie instanceof TaxonNameBase) {
 439                         if (langLabDet.keySet().contains(getTermService().getLanguageByIso("la"))) {
 440                                 labDet = langLabDet.get(getTermService().getLanguageByIso("la"));
 441                         } else if (langLabDet.keySet().contains(datasetLanguage)) {
 442                                 labDet = langLabDet.get(datasetLanguage);
 443                                 logger.info("TaxonName " + (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace) + " is not specified as a latin name.");
 444                         } else {
 445                                 labDet = langLabDet.get(langLabDet.keySet().iterator().next());
 446                                 logger.info("TaxonName " + (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace) + " is not specified as a latin name.");
 447                         }
 448                 } else {
 449                         labDet = langLabDet.get(langLabDet.keySet().iterator().next());
 450                 }
 451
 452                 //FIXME labDet is != null only for TaxonNameBase
 453                 ie.setTitleCache(labDet.get(0), true);
 454
 455                 if (labDet.size()>1) {
 456                         Annotation annotation = null;
 457                         if (labDet.get(1) != null) {
 458                                 if (labDet.get(2) != null) {
 459                                         annotation = Annotation.NewInstance(labDet.get(2) + " - " + labDet.get(1), datasetLanguage);
 460                                 } else {
 461                                         annotation = Annotation.NewInstance(labDet.get(1), datasetLanguage);
 462                                 }
 463                         }
 464                         ie.addAnnotation(annotation);
 465                 }
 466                 return;
 467         }
 468
 469         /**
 470          * @param elLabel
 471          * @return
 472          */
 473         private Language getLanguage(Element elLanguage) {
 474                 String lang = elLanguage.getAttributeValue("lang",xmlNamespace);
 475                 Language language = null;
 476                 if (StringUtils.isNotBlank(lang)) {
 477                         language = getTermService().getLanguageByIso(lang.substring(0, 2));
 478                 } else {
 479                         language = datasetLanguage;
 480                 }
 481                 return language;
 482         }
 483
 484
 485         // imports the representation (label, detail, lang) of a particular SDD element
 486         protected void importTechnicalMetadata(Element root, Namespace sddNamespace, SDDImportConfigurator sddConfig){
 487                 Element elTechnicalMetadata = root.getChild("TechnicalMetadata", sddNamespace);
 488                 String nameCreated = elTechnicalMetadata.getAttributeValue("created");
 489                 sourceReference = sddConfig.getSourceReference();
 490
 491                 if (nameCreated != null) {
 492                         if (!nameCreated.equals("")) {
 493                                 int year = Integer.parseInt(nameCreated.substring(0,4));
 494                                 int monthOfYear = Integer.parseInt(nameCreated.substring(5,7));
 495                                 int dayOfMonth = Integer.parseInt(nameCreated.substring(8,10));
 496                                 int hourOfDay = Integer.parseInt(nameCreated.substring(11,13));
 497                                 int minuteOfHour = Integer.parseInt(nameCreated.substring(14,16));
 498                                 int secondOfMinute = Integer.parseInt(nameCreated.substring(17,19));
 499                                 DateTime created = new DateTime(year,monthOfYear,dayOfMonth,hourOfDay,minuteOfHour,secondOfMinute,0);
 500                                 sourceReference.setCreated(created);
 501                                 sec.setCreated(created);
 502                         }
 503                 }
 504
 505                 // <Generator name="n/a, handcrafted instance document" version="n/a"/>
 506                 Element elGenerator = elTechnicalMetadata.getChild("Generator", sddNamespace);
 507                 generatorName = elGenerator.getAttributeValue("name");
 508                 generatorVersion = elGenerator.getAttributeValue("version");
 509
 510                 sec.addAnnotation(Annotation.NewDefaultLanguageInstance(generatorName + " - " + generatorVersion));
 511                 sourceReference.addAnnotation(Annotation.NewDefaultLanguageInstance(generatorName + " - " + generatorVersion));
 512
 513         }
 514
 515         // imports the complete dataset information
 516         protected boolean importDataset(Element elDataset, Namespace sddNamespace, SDDImportState state){                       // <Dataset xml:lang="en-us">
 517                 boolean success = true;
 518                 SDDImportConfigurator sddConfig = state.getConfig();
 519
 520                 importDatasetLanguage(elDataset,sddConfig);
 521                 importDatasetRepresentation(elDataset, sddNamespace);
 522                 importRevisionData(elDataset, sddNamespace);
 523                 importIPRStatements(elDataset, sddNamespace, sddConfig);
 524                 importTaxonNames(elDataset, sddNamespace, sddConfig);
 525
 526                 importDescriptiveConcepts(elDataset, sddNamespace, sddConfig);
 527                 success &= importCharacters(elDataset, sddNamespace, sddConfig);
 528                 importCharacterTrees(elDataset, sddNamespace, sddConfig, success);
 529
 530                 MarkerType editorMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerEditor, "editor", "Editor", "edt");
 531                 MarkerType geographicAreaMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerSDDGeographicArea, "SDDGeographicArea", "SDDGeographicArea", "ga");
 532                 MarkerType descriptiveConceptMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerDescriptiveConcept, "DescriptiveConcept", "Descriptive Concept", "DC");
 533                 markerTypes.add(editorMarkerType);
 534                 markerTypes.add(geographicAreaMarkerType);
 535                 markerTypes.add(descriptiveConceptMarkerType);
 536
 537                 //saving of all imported data into the CDM db
 538                 saveVocabularies();
 539                 saveFeatures();
 540                 saveModifiers();
 541                 saveStates();
 542                 saveMarkerType();
 543                 saveAreas(geographicAreaMarkerType);
 544                 saveUnits();
 545                 saveStatisticalMeasure();
 546                 saveAnnotationType();
 547
 548                 success &= importCodedDescriptions(elDataset, sddNamespace, sddConfig);
 549                 importAgents(elDataset, sddNamespace, sddConfig, success);
 550                 importPublications(elDataset, sddNamespace, sddConfig, success);
 551                 importMediaObjects(elDataset, sddNamespace, sddConfig, success);
 552                 importTaxonHierarchies(elDataset, sddNamespace, sddConfig, success);
 553                 importGeographicAreas(elDataset, sddNamespace, sddConfig);
 554                 importSpecimens(elDataset,sddNamespace, sddConfig);
 555
 556
 557                 if ((authors != null)||(editors != null)) {
 558                         Team team = Team.NewInstance();
 559                         if (authors != null) {
 560                                 for (Person author : authors.values()){
 561                                         team.addTeamMember(author);
 562                                 }
 563                         }
 564                         if (editors != null) {
 565                                 Marker marker = Marker.NewInstance();
 566                                 marker.setMarkerType(editorMarkerType);
 567                                 for (Person editor : editors.values()){
 568                                         Person edit = editor;
 569                                         edit.addMarker(marker);
 570                                         team.addTeamMember(edit);
 571                                 }
 572                         }
 573                         sec.setAuthorTeam(team);
 574                         sourceReference.setAuthorTeam(team);
 575                 }
 576
 577                 if (copyright != null) {
 578                         sourceReference.addRights(copyright);
 579                         sec.addRights(copyright);
 580                 }
 581
 582                 // Returns a CdmApplicationController created by the values of this configuration.
 583                 IDescriptionService descriptionService = getDescriptionService();
 584
 585                 for (TaxonDescription taxonDescription : taxonDescriptions.values()){
 586                         // Persists a Description
 587                         descriptionService.save(taxonDescription);
 588                 }
 589
 590                 for (String ref : taxonDescriptions.keySet()){
 591                         TaxonDescription td = taxonDescriptions.get(ref);
 592                         if (citations.containsKey(ref)) {
 593                                 ReferenceBase publication = publications.get(citations.get(ref));
 594                                 if (locations.containsKey(ref)) {
 595                                         Annotation location = Annotation.NewInstance(locations.get(ref), datasetLanguage);
 596                                         AnnotationType annotationType = AnnotationType.NewInstance("", "location", "");
 597                                         annotationTypes.add(annotationType);
 598                                         location.setAnnotationType(annotationType);
 599                                         (publication).addAnnotation(location);
 600                                 }
 601                                 td.addDescriptionSource(publication);
 602                         }
 603                 }
 604                 logger.info("end makeTaxonDescriptions ...");
 605
 606                 if (descriptiveConcepts != null) {
 607                         for (Feature feature : descriptiveConcepts) {
 608                                 Marker marker = Marker.NewInstance();
 609                                 marker.setMarkerType(descriptiveConceptMarkerType);
 610                                 feature.addMarker(marker);
 611                         }
 612                 }
 613                 saveFeatures();
 614
 615                 for (ReferenceBase publication : publications.values()){
 616                         getReferenceService().save(publication);
 617                 }
 618
 619                 for (ReferenceBase source : sources){
 620                         getReferenceService().save(source);
 621                 }
 622
 623                 for (FeatureTree featureTree : featureTrees) {
 624                         getFeatureTreeService().save(featureTree);
 625                 }
 626                 for (TaxonomicTree taxonomicTree : taxonomicTrees) {
 627                         getTaxonTreeService().save(taxonomicTree);
 628                 }
 629                 for (Specimen specimen : specimens.values()) {
 630                         getOccurrenceService().save(specimen);
 631                 }
 632                 logger.info("end of persistence ...");
 633
 634                 return success;
 635         }
 636
 637         /**
 638          *
 639          */
 640         private void saveVocabularies() {
 641                 for (TermVocabulary vocabulary : vocabularies ){
 642                         getVocabularyService().save(vocabulary);
 643                 }
 644
 645         }
 646
 647         private void saveAnnotationType() {
 648                 for (AnnotationType annotationType: annotationTypes){
 649                         getTermService().save(annotationType);
 650                 }
 651         }
 652
 653         private void saveStatisticalMeasure() {
 654                 for (StatisticalMeasure sm : statisticalMeasures){
 655                         getTermService().save(sm);
 656                 }
 657         }
 658
 659         private void saveUnits() {
 660                 if (units != null) {
 661                         for (MeasurementUnit unit : units.values()){
 662                                 if (unit != null) {
 663                                         getTermService().save(unit);
 664                                 }
 665                         }
 666                 }
 667         }
 668
 669         private void saveAreas(MarkerType geographicAreaMarkerType) {
 670                 for (NamedArea area : namedAreas.values() ){
 671                         Marker marker = Marker.NewInstance();
 672                         marker.setMarkerType(geographicAreaMarkerType);
 673                         area.addMarker(marker);
 674                         getTermService().save(area);
 675                 }
 676         }
 677
 678         private void saveStates() {
 679                 for (State state : states.values() ){
 680                         getTermService().save(state);
 681                 }
 682         }
 683
 684         private void saveMarkerType() {
 685                 for (MarkerType markerType : markerTypes){
 686                         getTermService().save(markerType);
 687                 }
 688         }
 689
 690         private void saveModifiers() {
 691                 for (Modifier modifier : modifiers.values() ){
 692                         getTermService().save(modifier);
 693                 }
 694         }
 695
 696         private void saveFeatures() {
 697                 for (Feature feature : features.values() ){
 698                         getTermService().save(feature);
 699                 }
 700         }
 701
 702         // imports the default language of the dataset
 703         protected void importDatasetLanguage(Element elDataset, SDDImportConfigurator sddConfig){
 704                 String nameLang = elDataset.getAttributeValue("lang",xmlNamespace);
 705
 706                 if (!nameLang.equals("")) {
 707                         String iso = nameLang.substring(0, 2);
 708                         datasetLanguage = getTermService().getLanguageByIso(iso);
 709                 } else {
 710                         datasetLanguage = Language.DEFAULT();
 711                 }
 712                 if (datasetLanguage == null) {
 713                         datasetLanguage = Language.DEFAULT();
 714                 }
 715         }
 716
 717         // imports the specimens
 718         protected void importSpecimens(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig) {
 719                 logger.info("start Specimens ...");
 720                 /*      <Specimens>
 721                         <Specimen id="sp1">
 722                                 <Representation>
 723                                 <Label>TJM45337</Label>
 724                                 </Representation>
 725                         </Specimen>
 726                 </Specimens>
 727                  */
 728                 Element elSpecimens = elDataset.getChild("Specimens",sddNamespace);
 729                 if (elSpecimens != null){
 730                         List<Element> listSpecimens = elSpecimens.getChildren("Specimen", sddNamespace);
 731                         int j = 0;
 732                         for (Element elSpecimen : listSpecimens) {
 733                                 String id = elSpecimen.getAttributeValue("id");
 734                                 Specimen specimen = null;
 735                                 if (!id.equals("")) {
 736                                         specimen = Specimen.NewInstance();
 737                                         specimens.put(id,specimen);
 738                                         importRepresentation(elSpecimen, sddNamespace, specimen, id, sddConfig);
 739                                 }
 740                         }
 741
 742                 }
 743         }
 744
 745         // imports the revision data associated with the Dataset (authors, modifications)
 746         protected void importRevisionData(Element elDataset, Namespace sddNamespace){
 747                 // <RevisionData>
 748                 logger.info("start RevisionData ...");
 749                 Element elRevisionData = elDataset.getChild("RevisionData",sddNamespace);
 750                 if (elRevisionData != null){
 751                         // <Creators>
 752                         Element elCreators = elRevisionData.getChild("Creators",sddNamespace);
 753
 754                         // <Agent role="aut" ref="a1"/>
 755                         List<Element> listAgents = elCreators.getChildren("Agent", sddNamespace);
 756
 757                         int j = 0;
 758                         //for each Agent
 759                         for (Element elAgent : listAgents){
 760
 761                                 String role = elAgent.getAttributeValue("role");
 762                                 String ref = elAgent.getAttributeValue("ref");
 763                                 if (role.equals("aut")) {
 764                                         if(!ref.equals("")) {
 765                                                 authors.put(ref, null);
 766                                         }
 767                                 }
 768                                 if (role.equals("edt")) {
 769                                         if(!ref.equals("")) {
 770                                                 editors.put(ref, null);
 771                                         }
 772                                 }
 773                                 if ((++j % modCount) == 0){ logger.info("Agents handled: " + j);}
 774
 775                         }
 776
 777                         // <DateModified>2006-04-08T00:00:00</DateModified>
 778                         String stringDateModified = (String)ImportHelper.getXmlInputValue(elRevisionData, "DateModified",sddNamespace);
 779
 780                         if (stringDateModified != null) {
 781                                 SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss");
 782                                 Date d = null;
 783                                 try {
 784                                         d = sdf.parse(stringDateModified);
 785                                 } catch(Exception e) {
 786                                         System.err.println("Exception :");
 787                                         e.printStackTrace();
 788                                 }
 789
 790                                 DateTime updated = null;
 791                                 if (d != null) {
 792                                         updated = new DateTime(d);
 793                                         sourceReference.setUpdated(updated);
 794                                         sec.setUpdated(updated);
 795                                 }
 796                         }
 797                 }
 798         }
 799
 800         // imports ipr statements associated with a dataset
 801         protected void importIPRStatements(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig){
 802                 // <IPRStatements>
 803                 logger.info("start IPRStatements ...");
 804                 Element elIPRStatements = elDataset.getChild("IPRStatements",sddNamespace);
 805                 // <IPRStatement role="Copyright">
 806                 if (elIPRStatements != null) {
 807                         List<Element> listIPRStatements = elIPRStatements.getChildren("IPRStatement", sddNamespace);
 808                         int j = 0;
 809                         //for each IPRStatement
 810
 811                         for (Element elIPRStatement : listIPRStatements){
 812
 813                                 String role = elIPRStatement.getAttributeValue("role");
 814                                 // <Label xml:lang="en-au">(c) 2003-2006 Centre for Occasional Botany.</Label>
 815                                 Element elLabel = elIPRStatement.getChild("Label",sddNamespace);
 816                                 String lang = "";
 817                                 if (elLabel != null) {
 818                                         lang = elLabel.getAttributeValue("lang",xmlNamespace);
 819                                 }
 820                                 String label = (String)ImportHelper.getXmlInputValue(elIPRStatement, "Label",sddNamespace);
 821
 822                                 if (role.equals("Copyright")) {
 823                                         Language iprLanguage = null;
 824                                         if (lang != null) {
 825                                                 if (!lang.equals("")) {
 826                                                         iprLanguage = getTermService().getLanguageByIso(lang.substring(0, 2));
 827                                                 } else {
 828                                                         iprLanguage = datasetLanguage;
 829                                                 }
 830                                         }
 831                                         if (iprLanguage == null) {
 832                                                 iprLanguage = datasetLanguage;
 833                                         }
 834                                         copyright = Rights.NewInstance(label, iprLanguage);
 835                                 }
 836
 837                                 if (copyright != null) {
 838                                         sourceReference.addRights(copyright);
 839                                         sec.addRights(copyright);
 840                                 }
 841
 842                                 if ((++j % modCount) == 0){ logger.info("IPRStatements handled: " + j);}
 843
 844                         }
 845                 }
 846         }
 847
 848         // imports the taxon names
 849         protected void importTaxonNames(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig){
 850                 // <TaxonNames>
 851                 logger.info("start TaxonNames ...");
 852                 Element elTaxonNames = elDataset.getChild("TaxonNames",sddNamespace);
 853                 // <TaxonName id="t1" uri="urn:lsid:authority:namespace:my-own-id">
 854                 if (elTaxonNames != null) {
 855                         List<Element> listTaxonNames = elTaxonNames.getChildren("TaxonName", sddNamespace);
 856                         int j = 0;
 857                         //for each TaxonName
 858                         for (Element elTaxonName : listTaxonNames){
 859
 860                                 String id = elTaxonName.getAttributeValue("id");
 861                                 String uri = elTaxonName.getAttributeValue("uri");
 862
 863                                 NonViralName tnb = null;
 864                                 if (!id.equals("")) {
 865                                         tnb = NonViralName.NewInstance(null);
 866                                         IdentifiableSource source = null;
 867                                         if (uri != null) {
 868                                                 if (!uri.equals("")) {
 869                                                         source = IdentifiableSource.NewInstance(id, "TaxonName", ReferenceFactory.newGeneric(), uri);
 870                                                 }
 871                                         } else {
 872                                                 source = IdentifiableSource.NewInstance(id, "TaxonName");
 873                                         }
 874                                         tnb.addSource(source);
 875                                         taxonNameBases.put(id,tnb);
 876                                 }
 877
 878                                 // <Representation>
 879                                 // <Label xml:lang="la">Viola hederacea Labill.</Label>
 880                                 importRepresentation(elTaxonName, sddNamespace, tnb, id, sddConfig);
 881
 882                                 if ((++j % modCount) == 0){ logger.info("TaxonNames handled: " + j);}
 883
 884                         }
 885                 }
 886         }
 887
 888         // imports the characters (categorical, quantitative and text ; sequence characters not supported) which correspond to CDM Features
 889         protected boolean importCharacters(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig){
 890                 boolean success = true;
 891                 // <Characters>
 892                 logger.info("start Characters ...");
 893                 Element elCharacters = elDataset.getChild("Characters", sddNamespace);
 894
 895                 // <CategoricalCharacter id="c1">
 896                 if (elCharacters != null) {
 897                         success &= handleCategoricalData(sddNamespace, sddConfig, elCharacters);
 898                         success &= handleQuantitativeData(sddNamespace, sddConfig, elCharacters);
 899                         success &= handleTextCharacters(sddNamespace, sddConfig, elCharacters);
 900                 }
 901
 902                 /*for (Iterator<Feature> f = features.values().iterator() ; f.hasNext() ;){
 903                         featureSet.add(f.next()); //XIM Why this line ?
 904                 }*/
 905
 906                 return success;
 907
 908         }
 909
 910         /**
 911          * @param sddNamespace
 912          * @param sddConfig
 913          * @param success
 914          * @param elCharacters
 915          * @return
 916          */
 917         private boolean handleCategoricalData(Namespace sddNamespace, SDDImportConfigurator sddConfig, Element elCharacters) {
 918                 boolean success = true;
 919                 List<Element> elCategoricalCharacters = elCharacters.getChildren("CategoricalCharacter", sddNamespace);
 920                 int j = 0;
 921                 for (Element elCategoricalCharacter : elCategoricalCharacters){
 922                         try {
 923
 924                                 String idCC = elCategoricalCharacter.getAttributeValue("id");
 925                                 Feature categoricalCharacter = Feature.NewInstance();
 926                                 categoricalCharacter.setKindOf(Feature.DESCRIPTION());
 927                                 importRepresentation(elCategoricalCharacter, sddNamespace, categoricalCharacter, idCC, sddConfig);
 928                                 categoricalCharacter.setSupportsCategoricalData(true);
 929
 930                                 // <States>
 931                                 Element elStates = elCategoricalCharacter.getChild("States",sddNamespace);
 932
 933                                 // <StateDefinition id="s1">
 934                                 List<Element> elStateDefinitions = elStates.getChildren("StateDefinition",sddNamespace);
 935                                 TermVocabulary<State> termVocabularyState = new TermVocabulary<State>();
 936
 937                                 vocabularies.add(termVocabularyState);
 938
 939                                 int k = 0;
 940                                 //for each StateDefinition
 941                                 for (Element elStateDefinition : elStateDefinitions){
 942
 943                                         if ((++k % modCount) == 0){ logger.info("StateDefinitions handled: " + (k-1));}
 944
 945                                         String idS = elStateDefinition.getAttributeValue("id");
 946                                         State state = states.get(idS);
 947                                         if (state == null){
 948                                                 state = State.NewInstance();
 949                                         }else{
 950                                                 logger.debug("State duplicate found");
 951                                         }
 952                                         importRepresentation(elStateDefinition, sddNamespace, state, idS, sddConfig);
 953
 954                                         termVocabularyState.addTerm(state);
 955                                         states.put(idS,state);
 956                                 }
 957                                 categoricalCharacter.addSupportedCategoricalEnumeration(termVocabularyState);
 958                                 features.put(idCC, categoricalCharacter);
 959
 960                         } catch (Exception e) {
 961                                 logger.warn("Import of CategoricalCharacter " + j + " failed.");
 962                                 success = false;
 963                         }
 964
 965                         if ((++j % modCount) == 0){ logger.info("CategoricalCharacters handled: " + j);}
 966
 967                 }
 968                 return success;
 969         }
 970
 971         /**
 972          * @param sddNamespace
 973          * @param sddConfig
 974          * @param elCharacters
 975          */
 976         private boolean handleQuantitativeData(Namespace sddNamespace,  SDDImportConfigurator sddConfig, Element elCharacters) {
 977                 boolean success = true;
 978                 int j;
 979                 // <QuantitativeCharacter id="c2">
 980                 List<Element> elQuantitativeCharacters = elCharacters.getChildren("QuantitativeCharacter", sddNamespace);
 981                 j = 0;
 982                 //for each QuantitativeCharacter
 983                 for (Element elQuantitativeCharacter : elQuantitativeCharacters){
 984
 985                         try {
 986
 987                                 String idQC = elQuantitativeCharacter.getAttributeValue("id");
 988
 989                                 // <Representation>
 990                                 //  <Label>Leaf length</Label>
 991                                 // </Representation>
 992                                 Feature quantitativeCharacter = Feature.NewInstance();
 993                                 quantitativeCharacter.setKindOf(Feature.DESCRIPTION());
 994                                 importRepresentation(elQuantitativeCharacter, sddNamespace, quantitativeCharacter, idQC, sddConfig);
 995
 996                                 quantitativeCharacter.setSupportsQuantitativeData(true);
 997
 998                                 // <MeasurementUnit>
 999                                 //  <Label role="Abbrev">m</Label>
1000                                 // </MeasurementUnit>
1001                                 Element elMeasurementUnit = elQuantitativeCharacter.getChild("MeasurementUnit",sddNamespace);
1002                                 String label = "";
1003                                 String role = "";
1004                                 if (elMeasurementUnit != null) {
1005                                         Element elLabel = elMeasurementUnit.getChild("Label",sddNamespace);
1006                                         role = elLabel.getAttributeValue("role");
1007                                         label = (String)ImportHelper.getXmlInputValue(elMeasurementUnit, "Label",sddNamespace);
1008                                 }
1009
1010                                 MeasurementUnit unit = null;
1011                                 if (!label.equals("")){
1012                                         if (role != null) {
1013                                                 if (role.equals("Abbrev")){
1014                                                         unit = MeasurementUnit.NewInstance(label,label,label);
1015                                                 }
1016                                         } else {
1017                                                 unit = MeasurementUnit.NewInstance(label,label,label);
1018                                         }
1019                                 }
1020
1021                                 if (unit != null) {
1022                                         units.put(idQC, unit);
1023                                 }
1024
1025                                 //<Default>
1026                                 //  <MeasurementUnitPrefix>milli</MeasurementUnitPrefix>
1027                                 //</Default>
1028                                 Element elDefault = elQuantitativeCharacter.getChild("Default",sddNamespace);
1029                                 if (elDefault != null) {
1030                                         String measurementUnitPrefix = (String)ImportHelper.getXmlInputValue(elDefault, "MeasurementUnitPrefix",sddNamespace);
1031                                         if (! measurementUnitPrefix.equals("")){
1032                                                 defaultUnitPrefixes.put(idQC, measurementUnitPrefix);
1033                                         }
1034                                 }
1035
1036                                 features.put(idQC, quantitativeCharacter);
1037
1038                         } catch (Exception e) {
1039                                 //FIXME
1040                                 logger.warn("Import of QuantitativeCharacter " + j + " failed.");
1041                                 success = false;
1042                         }
1043
1044                         if ((++j % modCount) == 0){ logger.info("QuantitativeCharacters handled: " + j);}
1045
1046                 }
1047                 return success;
1048         }
1049
1050         private boolean handleTextCharacters(Namespace sddNamespace, SDDImportConfigurator sddConfig, Element elCharacters) {
1051                 boolean success = true;
1052                 int j;
1053                 // <TextCharacter id="c3">
1054                 List<Element> elTextCharacters = elCharacters.getChildren("TextCharacter", sddNamespace);
1055                 j = 0;
1056                 //for each TextCharacter
1057                 for (Element elTextCharacter : elTextCharacters){
1058
1059                         try {
1060
1061                                 String idTC = elTextCharacter.getAttributeValue("id");
1062
1063                                 // <Representation>
1064                                 //  <Label xml:lang="en">Leaf features not covered by other characters</Label>
1065                                 // </Representation>
1066                                 Feature textCharacter = Feature.NewInstance();
1067                                 textCharacter.setKindOf(Feature.DESCRIPTION());
1068                                 importRepresentation(elTextCharacter, sddNamespace, textCharacter, idTC, sddConfig);
1069
1070                                 textCharacter.setSupportsTextData(true);
1071
1072                                 features.put(idTC, textCharacter);
1073
1074                         } catch (Exception e) {
1075                                 //FIXME
1076                                 logger.warn("Import of TextCharacter " + j + " failed.");
1077                                 success = false;
1078                         }
1079
1080                         if ((++j % modCount) == 0){ logger.info("TextCharacters handled: " + j);}
1081
1082                 }
1083                 return success;
1084         }
1085
1086         // imports the descriptions of taxa
1087         protected boolean importCodedDescriptions(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig){
1088                 boolean success = true;
1089
1090                 // <CodedDescriptions>
1091                 logger.info("start CodedDescriptions ...");
1092                 Element elCodedDescriptions = elDataset.getChild("CodedDescriptions",sddNamespace);
1093
1094                 // <CodedDescription id="D101">
1095                 if (elCodedDescriptions != null) {
1096                         List<Element> listCodedDescriptions = elCodedDescriptions.getChildren("CodedDescription", sddNamespace);
1097                         int j = 0;
1098                         //for each CodedDescription
1099                         for (Element elCodedDescription : listCodedDescriptions){
1100                                 success &= handleCodedDescription(sddNamespace, sddConfig, elCodedDescription, j);
1101                                 if ((++j % modCount) == 0){ logger.info("CodedDescriptions handled: " + j);}
1102                         }
1103                 }
1104                 return success;
1105         }
1106
1107         /**
1108          * @param sddNamespace
1109          * @param sddConfig
1110          * @param j
1111          * @param elCodedDescription
1112          * @return
1113          */
1114         private boolean handleCodedDescription(Namespace sddNamespace, SDDImportConfigurator sddConfig, Element elCodedDescription, int j) {
1115                 boolean success = true ;
1116                 try {
1117
1118                         String idCD = elCodedDescription.getAttributeValue("id");
1119
1120                         // <Representation>
1121                         //  <Label>&lt;i&gt;Viola hederacea&lt;/i&gt; Labill. as revised by R. Morris April 8, 2006</Label>
1122                         // </Representation>
1123                         TaxonDescription taxonDescription = TaxonDescription.NewInstance();
1124                         importRepresentation(elCodedDescription, sddNamespace, taxonDescription, idCD, sddConfig);
1125
1126                         // <Scope>
1127                         //  <TaxonName ref="t1"/>
1128                         //  <Citation ref="p1" location="p. 30"/>
1129                         // </Scope>
1130                         Element elScope = elCodedDescription.getChild("Scope", sddNamespace);
1131                         Taxon taxon;
1132                         if (elScope != null) {
1133                                 taxon = handleCDScope(sddNamespace, sddConfig, idCD, elScope);
1134                         } else {//in case no taxon is linked to the description, a new one is created
1135                                 taxon = handleCDNoScope(sddNamespace, sddConfig, elCodedDescription);
1136                         }
1137
1138                         // <SummaryData>
1139                         Element elSummaryData = elCodedDescription.getChild("SummaryData",sddNamespace);
1140                         if (elSummaryData != null) {
1141                                 handleSummaryCategoricalData(sddNamespace, taxonDescription, elSummaryData);
1142                                 handleSummaryQuantitativeData(sddNamespace, taxonDescription, elSummaryData);
1143                                 handleSummaryTextData(sddNamespace, taxonDescription, elSummaryData);
1144                         }
1145
1146                         if (taxon != null) {
1147                                 taxon.addDescription(taxonDescription);
1148                         }
1149 //
1150                         taxonDescription.setDescriptiveSystem(featureSet);
1151
1152                         taxonDescriptions.put(idCD, taxonDescription);//FIXME
1153
1154                 } catch (Exception e) {
1155                         //FIXME
1156                         logger.warn("Import of CodedDescription " + j + " failed.", e);
1157                         success = false;
1158                 }
1159                 return success;
1160         }
1161
1162         /**
1163          * @param sddNamespace
1164          * @param sddConfig
1165          * @param elCodedDescription
1166          * @param taxon
1167          * @return
1168          */
1169         private Taxon handleCDNoScope(Namespace sddNamespace,
1170                         SDDImportConfigurator sddConfig, Element elCodedDescription     ) {
1171                 Taxon taxon = null;
1172                 NonViralName nonViralName = NonViralName.NewInstance(null);
1173                 String id = new String("" + taxonNamesCount);
1174                 IdentifiableSource source = IdentifiableSource.NewInstance(id, "TaxonName");
1175                 importRepresentation(elCodedDescription, sddNamespace, nonViralName, id, sddConfig);
1176
1177                 if(sddConfig.isDoMatchTaxa()){
1178                         taxon = getTaxonService().findBestMatchingTaxon(nonViralName.getTitleCache());
1179                 }
1180
1181                 if(taxon != null){
1182                         nonViralName = HibernateProxyHelper.deproxy(taxon.getName(), NonViralName.class);
1183 //                                                      taxonNameBases.put(id ,tnb);
1184 //                                                      taxonNamesCount++;
1185                         logger.info("using existing Taxon " + taxon.getTitleCache());
1186                 } else {
1187                         nonViralName.addSource(source);
1188                         taxonNameBases.put(id ,nonViralName);
1189                         taxonNamesCount++;
1190                         logger.info("creating new Taxon from TaxonName " + nonViralName.getTitleCache());
1191                         taxon = Taxon.NewInstance(nonViralName, sec);
1192                 }
1193                 return taxon;
1194         }
1195
1196         /**
1197          * @param sddNamespace
1198          * @param sddConfig
1199          * @param idCD
1200          * @param elScope
1201          * @param taxon
1202          * @return
1203          */
1204         private Taxon handleCDScope(Namespace sddNamespace, SDDImportConfigurator sddConfig,
1205                         String idCD, Element elScope) {
1206                 Taxon taxon = null;
1207                 Element elTaxonName = elScope.getChild("TaxonName", sddNamespace);
1208                 String ref = elTaxonName.getAttributeValue("ref");
1209                 NonViralName nonViralName = taxonNameBases.get(ref);
1210
1211                 if(sddConfig.isDoMatchTaxa()){
1212                         taxon = getTaxonService().findBestMatchingTaxon(nonViralName.getTitleCache());
1213                 }
1214
1215                 if(taxon != null){
1216                         logger.info("using existing Taxon" + taxon.getTitleCache());
1217                         if(!nonViralName.getUuid().equals(taxon.getName().getUuid())){
1218                                 logger.warn("TaxonNameBase entity of existing taxon does not match Name in list -> replacing Name in list");
1219                                 nonViralName = HibernateProxyHelper.deproxy(taxon.getName(), NonViralName.class);
1220                         }
1221                 } else {
1222                         logger.info("creating new Taxon from TaxonName" + nonViralName.getTitleCache());
1223                         taxon = Taxon.NewInstance(nonViralName, sec);
1224                 }
1225
1226                 //citation
1227                 Element elCitation = elScope.getChild("Citation",sddNamespace);
1228                 if (elCitation != null) {
1229                         String refCitation = elCitation.getAttributeValue("ref");
1230                         if (! refCitation.equals("")){
1231                                 citations.put(idCD, refCitation);
1232                         }
1233                         String location = elCitation.getAttributeValue("location");
1234                         if (! location.equals("")){
1235                                 locations.put(idCD, location);
1236                         }
1237                 }
1238                 return taxon;
1239         }
1240
1241         /**
1242          * @param sddNamespace
1243          * @param taxonDescription
1244          * @param elSummaryData
1245          */
1246         private void handleSummaryTextData(Namespace sddNamespace,
1247                         TaxonDescription taxonDescription, Element elSummaryData) {
1248                 String ref;
1249                 int k;
1250                 // <TextChar ref="c3">
1251                 List<Element> elTextChars = elSummaryData.getChildren("TextChar", sddNamespace);
1252                 k = 0;
1253                 //for each TextChar
1254                 for (Element elTextChar : elTextChars){
1255                         if ((++k % modCount) == 0){ logger.info("TextChar handled: " + (k-1));}
1256                         ref = elTextChar.getAttributeValue("ref");
1257                         Feature feature = features.get(ref);
1258                         TextData textData = TextData.NewInstance();
1259                         textData.setFeature(feature);
1260
1261                         // <Content>Free form text</Content>
1262                         String content = (String)ImportHelper.getXmlInputValue(elTextChar, "Content",sddNamespace);
1263                         textData.putText(content, datasetLanguage);
1264                         taxonDescription.addElement(textData);
1265                 }
1266         }
1267
1268         /**
1269          * @param sddNamespace
1270          * @param taxonDescription
1271          * @param elSummaryData
1272          */
1273         private void handleSummaryQuantitativeData(Namespace sddNamespace,
1274                         TaxonDescription taxonDescription, Element elSummaryData) {
1275                 String ref;
1276                 int k;
1277                 // <Quantitative ref="c2">
1278                 List<Element> elQuantitatives = elSummaryData.getChildren("Quantitative", sddNamespace);
1279                 k = 0;
1280                 //for each Quantitative
1281                 for (Element elQuantitative : elQuantitatives){
1282                         if ((++k % modCount) == 0){ logger.warn("Quantitative handled: " + (k-1));}
1283                         ref = elQuantitative.getAttributeValue("ref");
1284                         Feature feature = features.get(ref);
1285                         QuantitativeData quantitativeData = QuantitativeData.NewInstance();
1286                         quantitativeData.setFeature(feature);
1287
1288                         MeasurementUnit unit = units.get(ref);
1289                         String prefix = defaultUnitPrefixes.get(ref);
1290                         if (unit != null) {
1291                                 String u = unit.getLabel();
1292                                 if (prefix != null) {
1293                                         u = prefix + u;
1294                                 }
1295                                 unit.setLabel(u);
1296                                 quantitativeData.setUnit(unit);
1297                         }
1298
1299                         // <Measure type="Min" value="2.3"/>
1300                         List<Element> elMeasures = elQuantitative.getChildren("Measure", sddNamespace);
1301                         int l = 0;
1302
1303                         //for each State
1304                         for (Element elMeasure : elMeasures){
1305                                 if ((++l % modCount) == 0){ logger.info("States handled: " + (l-1));}
1306                                 String type = elMeasure.getAttributeValue("type");
1307                                 String value = elMeasure.getAttributeValue("value");
1308                                 if (value.contains(",")) {
1309                                         value = value.replace(',', '.');
1310                                 }
1311                                 Float v = Float.parseFloat(value);
1312                                 //Float v = new Float(0);
1313                                 StatisticalMeasure t = null;
1314                                 if (type.equals("Min")) {
1315                                         t = StatisticalMeasure.MIN();
1316                                 } else if (type.equals("Mean")) {
1317                                         t = StatisticalMeasure.AVERAGE();
1318                                 } else if (type.equals("Max")) {
1319                                         t = StatisticalMeasure.MAX();
1320                                 } else if (type.equals("SD")) {
1321                                         t = StatisticalMeasure.STANDARD_DEVIATION();
1322                                 } else if (type.equals("N")) {
1323                                         t = StatisticalMeasure.SAMPLE_SIZE();
1324                                 } else if (type.equals("UMethLower")) {
1325                                         t = StatisticalMeasure.TYPICAL_LOWER_BOUNDARY();
1326                                 } else if (type.equals("UMethUpper")) {
1327                                         t = StatisticalMeasure.TYPICAL_UPPER_BOUNDARY();
1328                                 } else if (type.equals("Var")) {
1329                                         t = StatisticalMeasure.VARIANCE();
1330                                 } else {
1331                                         t = StatisticalMeasure.NewInstance(type,type,type);
1332                                         statisticalMeasures.add(t);
1333                                 }
1334
1335                                 StatisticalMeasurementValue statisticalValue = StatisticalMeasurementValue.NewInstance();
1336                                 statisticalValue.setValue(v);
1337                                 statisticalValue.setType(t);
1338                                 quantitativeData.addStatisticalValue(statisticalValue);
1339                                 featureData.add(statisticalValue);
1340                         }
1341                         taxonDescription.addElement(quantitativeData);
1342                 }
1343         }
1344
1345         /**
1346          * @param sddNamespace
1347          * @param taxonDescription
1348          * @param elSummaryData
1349          */
1350         private void handleSummaryCategoricalData(Namespace sddNamespace,
1351                         TaxonDescription taxonDescription, Element elSummaryData) {
1352                 String ref;
1353                 // <Categorical ref="c4">
1354                 List<Element> elCategoricals = elSummaryData.getChildren("Categorical", sddNamespace);
1355                 int k = 0;
1356                 //for each Categorical
1357                 for (Element elCategorical : elCategoricals){
1358                         if ((++k % modCount) == 0){ logger.warn("Categorical handled: " + (k-1));}
1359                         ref = elCategorical.getAttributeValue("ref");
1360                         Feature feature = features.get(ref);
1361                         CategoricalData categoricalData = CategoricalData.NewInstance();
1362                         categoricalData.setFeature(feature);
1363
1364                         // <State ref="s3"/>
1365                         List<Element> elStates = elCategorical.getChildren("State", sddNamespace);
1366                         int l = 0;
1367
1368                         //for each State
1369                         for (Element elState : elStates){
1370                                 if ((++l % modCount) == 0){ logger.info("States handled: " + (l-1));}
1371                                 ref = elState.getAttributeValue("ref");
1372                                 State state = states.get(ref);
1373                                 if (state != null) {
1374                                         StateData stateData = StateData.NewInstance();
1375                                         stateData.setState(state);
1376                                         List<Element> elModifiers = elState.getChildren("Modifier", sddNamespace);
1377                                         for (Element elModifier : elModifiers){
1378                                                 ref = elModifier.getAttributeValue("ref");
1379                                                 Modifier modifier = modifiers.get(ref);
1380                                                 if (modifier != null) {
1381                                                         stateData.addModifier(modifier);
1382                                                 }
1383                                         }
1384                                         categoricalData.addState(stateData);
1385                                 }
1386                                 taxonDescription.addElement(categoricalData);
1387                         }
1388                 }
1389         }
1390
1391         // imports the persons associated with the dataset creation, modification, related publications
1392         protected void importAgents(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig, boolean success){
1393                 // <Agents>
1394                 logger.info("start Agents ...");
1395                 Element elAgents = elDataset.getChild("Agents",sddNamespace);
1396                 if (elAgents != null) {
1397                         // <Agent id="a1">
1398                         List <Element> listAgents = elAgents.getChildren("Agent", sddNamespace);
1399                         int j = 0;
1400                         //for each Agent
1401                         for (Element elAgent : listAgents){
1402
1403                                 try {
1404
1405                                         String idA = elAgent.getAttributeValue("id");
1406
1407                                         //  <Representation>
1408                                         //   <Label>Kevin Thiele</Label>
1409                                         //   <Detail role="Description">Ali Baba is also known as r.a.m.</Detail>
1410                                         //  </Representation>
1411                                         Person person = Person.NewInstance();
1412                                         importRepresentation(elAgent, sddNamespace, person, idA, sddConfig);
1413                                         person.addSource(IdentifiableSource.NewInstance(idA, "Agent"));
1414
1415                                         /*XIM <Links>
1416                                         Element elLinks = elAgent.getChild("Links",sddNamespace);
1417
1418                                         if (elLinks != null) {
1419
1420                                                 //  <Link rel="Alternate" href="http://www.diversitycampus.net/people/hagedorn"/>
1421                                                 List<Element> listLinks = elLinks.getChildren("Link", sddNamespace);
1422                                                 int k = 0;
1423                                                 //for each Link
1424                                                 for (Element elLink : listLinks){
1425
1426                                                         try {
1427
1428                                                                 String rel = elLink.getAttributeValue("rel");
1429                                                                 String href = elLink.getAttributeValue("href");
1430
1431                                                                 Media link = Media.NewInstance();
1432                                                                 MediaRepresentation mr = MediaRepresentation.NewInstance();
1433                                                                 mr.addRepresentationPart(MediaRepresentationPart.NewInstance(href, null));
1434                                                                 link.addRepresentation(mr);
1435                                                                 person.addMedia(link);
1436
1437                                                         } catch (Exception e) {
1438                                                                 //FIXME
1439                                                                 logger.warn("Import of Link " + k + " failed.");
1440                                                                 success = false;
1441                                                         }
1442
1443                                                         if ((++k % modCount) == 0){ logger.info("Links handled: " + k);}
1444
1445                                                 }
1446                                         }
1447                                         */
1448                                         if (authors.containsKey(idA)) {
1449                                                 authors.put(idA,person);
1450                                         }
1451
1452                                         if (editors.containsKey(idA)) {
1453                                                 editors.put(idA, person);
1454                                         }
1455
1456                                 } catch (Exception e) {
1457                                         //FIXME
1458                                         logger.warn("Import of Agent " + j + " failed.");
1459                                         success = false;
1460                                 }
1461
1462                                 if ((++j % modCount) == 0){ logger.info("Agents handled: " + j);}
1463
1464                         }
1465                 }
1466         }
1467
1468         // imports publications related with the data set
1469         protected void importPublications(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig, boolean success){
1470                 /* <Publications>
1471                           <Publication id="p112">
1472                             <Representation>
1473                               <Label>Gee, X. & Haa, Y. (2003). How to be happy in five minutes. Instant Gratifications, Palm Beach.</Label>
1474                             </Representation>
1475                             <Links>
1476                             <Link rel="BasedOn" href="doi:10.1992/32311"/>
1477                             <Link rel="Alternate" href="http://some.service.net/providing/bibliographic.data"/>
1478                             </Links>
1479                         </Publications>
1480 */
1481                 logger.info("start Publications ...");
1482                 Element elPublications = elDataset.getChild("Publications",sddNamespace);
1483
1484                 if (elPublications != null) {
1485                         List<Element> listPublications = elPublications.getChildren("Publication", sddNamespace);
1486                         int j = 0;
1487                         for (Element elPublication : listPublications){
1488
1489                                 try {
1490
1491                                         String idP = elPublication.getAttributeValue("id");
1492                                         ReferenceBase publication = ReferenceFactory.newArticle();
1493                                         importRepresentation(elPublication, sddNamespace, publication, idP, sddConfig);
1494
1495                                         publications.put(idP,publication);
1496
1497                                 } catch (Exception e) {
1498                                         logger.warn("Import of Publication " + j + " failed.");
1499                                         success = false;
1500                                 }
1501
1502                                 if ((++j % modCount) == 0){ logger.info("Publications handled: " + j);}
1503
1504                         }
1505                 }
1506         }
1507
1508         // imports media objects such as images //FIXME check mediaobj
1509         protected void importMediaObjects(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig, boolean success){
1510                 // <MediaObjects>
1511                 logger.info("start MediaObjects ...");
1512                 Element elMediaObjects = elDataset.getChild("MediaObjects",sddNamespace);
1513
1514                 if (elMediaObjects != null) {
1515                         // <MediaObject id="m1">
1516                         List<Element> listMediaObjects = elMediaObjects.getChildren("MediaObject", sddNamespace);
1517                         int j = 0;
1518                         for (Element elMO : listMediaObjects){
1519
1520                                 String id = "";
1521
1522                                 try {
1523                                         String idMO = elMO.getAttributeValue("id");
1524                                         id = idMO;
1525
1526                                         //  <Representation>
1527                                         //   <Label>Image description, e.g. to be used for alt-attribute in html.</Label>
1528                                         //  </Representation>
1529                                         Media media = Media.NewInstance();
1530                                         importRepresentation(elMO, sddNamespace, media, idMO, sddConfig);
1531
1532                                         // <Type>Image</Type>
1533                                         // <Source href="http://test.edu/test.jpg"/>
1534                                         String type = (String)ImportHelper.getXmlInputValue(elMO,"Type",sddNamespace);
1535
1536                                         if ((type != null) && (type.equals("Image"))) {
1537                                                 Element elSource = elMO.getChild("Source",sddNamespace);
1538                                                 String href = elSource.getAttributeValue("href");
1539
1540                                                 ImageMetaData imageMetaData = ImageMetaData.newInstance();
1541                                                 ImageFile image = null;
1542                                                 if (href.substring(0,7).equals("http://")) {
1543                                                         try{
1544                                                                 URL url = new URL(href);
1545
1546                                                                 imageMetaData.readMetaData(url.toURI(), 0);
1547                                                                 image = ImageFile.NewInstance(url.toString(), null, imageMetaData);
1548                                                         } catch (MalformedURLException e) {
1549                                                                 logger.error("Malformed URL", e);
1550                                                         }
1551                                                 } else {
1552                                                         String sns = sddConfig.getSourceNameString();
1553                                                         File f = new File(sns);
1554                                                         File parent = f.getParentFile();
1555                                                         String fi = parent.toString() + File.separator + href;
1556                                                         File file = new File(fi);
1557                                                         imageMetaData.readMetaData(new URI(fi), 0); //file
1558                                                         image = ImageFile.NewInstance(file.toString(), null, imageMetaData);
1559                                                 }
1560                                                 MediaRepresentation representation = MediaRepresentation.NewInstance(imageMetaData.getMimeType(), null);
1561                                                 representation.addRepresentationPart(image);
1562
1563                                                 media.addRepresentation(representation);
1564
1565                                                 ArrayList<CdmBase> lcb = (ArrayList<CdmBase>) mediaObject_ListCdmBase.get(idMO);
1566                                                 if (lcb != null) {
1567                                                         for (int k = 0; k < lcb.size(); k++) {
1568                                                                 if (lcb.get(k) instanceof DefinedTermBase) {
1569                                                                         DefinedTermBase dtb = (DefinedTermBase) lcb.get(k);
1570                                                                         // if (lcb.get(0) instanceof DefinedTermBase) {
1571                                                                         // DefinedTermBase dtb = (DefinedTermBase) lcb.get(0);
1572                                                                         //                                                                      if (dtb!=null) {
1573                                                                         //                                                                              if (k == 0) {
1574                                                                         dtb.addMedia(media);
1575                                                                         //System.out.println(dtb.getLabel());
1576                                                                         //                                                                              } else {
1577                                                                         //                                                                                      Media me = (Media) media.clone();
1578                                                                         //                                                                                      dtb.addMedia(me);
1579                                                                         //                                                                              }
1580                                                                         //                                                                      }
1581                                                                 } else if (lcb.get(k) instanceof ReferenceBase) {
1582                                                                         ReferenceBase rb = (ReferenceBase) lcb.get(k);
1583                                                                         //} else if (lcb.get(0) instanceof ReferenceBase) {
1584                                                                         //ReferenceBase rb = (ReferenceBase) lcb.get(0);
1585                                                                         // rb.setTitleCache(label);
1586                                                                         //                                                                      if (rb!=null) {
1587                                                                         //                                                                              if (k == 0) {
1588                                                                         rb.addMedia(media);
1589                                                                         //System.out.println(rb.getTitle());
1590                                                                         //                                                                              } else {
1591                                                                         //                                                                                      Media me = (Media) media.clone();
1592                                                                         //                                                                                      rb.addMedia(me);
1593                                                                         //                                                                              }
1594                                                                         //                                                                      }
1595                                                                 }/* else if (lcb.get(k) instanceof TaxonNameBase){
1596                                                                         TaxonNameBase tb = (TaxonNameBase) lcb.get(k);
1597                                                                         tb.addMedia(media);
1598                                                         }*/
1599                                                         }
1600                                                 }
1601                                         }
1602
1603                                 } catch (Exception e) {
1604                                         //FIXME
1605                                         logger.warn("Could not attach MediaObject " + j + "(SDD: " + id + ") to several objects.");
1606                                         success = false;
1607                                 }
1608
1609                                 if ((++j % modCount) == 0){ logger.info("MediaObjects handled: " + j);
1610
1611                                 }
1612                         }
1613                 }
1614         }
1615
1616         // imports the <DescriptiveConcepts> block ; DescriptiveConcepts are used as nodes in CharacterTrees and Characters as leaves
1617         // but since Modifiers can be linked to DescriptiveConcepts they are stored as features with a particular Marker
1618         protected void importDescriptiveConcepts(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig){
1619                 /* <DescriptiveConcepts>
1620                       <DescriptiveConcept id="dc0">
1621                                 <Representation>
1622                                   <Label>Fixed set of modifiers supported in Lucid3</Label>
1623                                 </Representation>
1624                                 <Modifiers>
1625                                   <Modifier id="mod1">
1626                                     <Representation>
1627                                       <Label>rarely</Label>
1628                                     </Representation>
1629                                     <ModifierClass>Frequency</ModifierClass>
1630                                     <ProportionRange lowerestimate="0.0" upperestimate="0.25"/>
1631                                   </Modifier>
1632                           </Modifiers>
1633                         </DescriptiveConcept>
1634                  </DescriptiveConcepts>
1635                  */
1636                 logger.info("start DescriptiveConcepts ...");
1637                 Element elDescriptiveConcepts = elDataset.getChild("DescriptiveConcepts",sddNamespace);
1638                 if (elDescriptiveConcepts != null) {
1639                         List<Element> listDescriptiveConcepts = elDescriptiveConcepts.getChildren("DescriptiveConcept", sddNamespace);
1640                         int j = 0;
1641
1642                         for (Element elDescriptiveConcept : listDescriptiveConcepts){
1643                                 try {
1644                                 String id = elDescriptiveConcept.getAttributeValue("id");
1645                                         Feature feature = Feature.NewInstance();
1646                                         feature.setKindOf(Feature.DESCRIPTION());
1647                                         if (!id.equals("")) {
1648                                         //       <Representation>
1649                                         //       <Label>Body</Label>
1650                                         importRepresentation(elDescriptiveConcept, sddNamespace, feature, id, sddConfig);
1651                                                 features.put(id, feature);
1652                                                 getTermService().save(feature);//XIM
1653                                                 descriptiveConcepts.add(feature);
1654                                                 // imports the modifiers
1655                                                 Element elModifiers = elDescriptiveConcept.getChild("Modifiers", sddNamespace);
1656                                         if (elModifiers !=null){
1657                                                 List<Element> listModifiers = elModifiers.getChildren("Modifier", sddNamespace);
1658                                                         TermVocabulary<Modifier> termVocabularyState = new TermVocabulary<Modifier>();
1659                                                 for (Element elModifier : listModifiers) {
1660                                                                 Modifier modif = Modifier.NewInstance();
1661                                                                 String idmod = elModifier.getAttributeValue("id");
1662                                                                 importRepresentation(elModifier, sddNamespace, modif, idmod, sddConfig);
1663                                                                 termVocabularyState.addTerm(modif);
1664                                                                 //termVocabularyStates.add(termVocabularyState);
1665                                                                 getVocabularyService().save(termVocabularyState);//XIM
1666                                                                 modifiers.put(idmod, modif);
1667                                                 }
1668                                                         feature.addRecommendedModifierEnumeration(termVocabularyState);
1669                                 }
1670
1671                                         }
1672                                 }
1673                                 catch (Exception e) {
1674                                         logger.warn("Import of DescriptiveConcept " + j + " failed: " + e.getMessage());
1675                                 }
1676                                 if ((++j % modCount) == 0){ logger.info("DescriptiveConcepts handled: " + j);}
1677
1678                         }
1679                 }
1680         }
1681
1682         // imports the <CharacterTrees> block
1683         protected void importCharacterTrees(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig, boolean success){
1684                 // <CharacterTrees>
1685                 logger.info("start CharacterTrees ...");
1686                 Element elCharacterTrees = elDataset.getChild("CharacterTrees",sddNamespace);
1687
1688                 if (elCharacterTrees != null) {
1689                         List<Element> listCharacterTrees = elCharacterTrees.getChildren("CharacterTree", sddNamespace);
1690                         int j = 0;
1691                         for (Element elCharacterTree : listCharacterTrees){
1692                                 try {
1693                                         Element elRepresentation = elCharacterTree.getChild("Representation",sddNamespace);
1694                                         String label = (String)ImportHelper.getXmlInputValue(elRepresentation,"Label",sddNamespace);
1695                                         //Element elDesignedFor = elCharacterTree.getChild("DesignedFor",sddNamespace);//TODO ?
1696
1697                                         FeatureTree feattree =  FeatureTree.NewInstance();
1698                                         importRepresentation(elCharacterTree, sddNamespace, feattree, "", sddConfig);
1699                                         FeatureNode root = feattree.getRoot();
1700                                         List<Element> listelNodes = elCharacterTree.getChildren("Nodes", sddNamespace);
1701
1702                                         //Nodes of CharacterTrees in SDD always refer to DescriptiveConcepts
1703                                         for (Element elNodes : listelNodes) {
1704                                                 handleCharacterNodes(sddNamespace, root, elNodes);
1705                                         }
1706                                         featureTrees.add(feattree);
1707                                 }
1708
1709                                 catch (Exception e) {
1710                                         logger.warn("Import of Character tree " + j + " failed.");
1711                                         success = false;
1712                                 }
1713                                 if ((++j % modCount) == 0){ logger.info("CharacterTrees handled: " + j);}
1714
1715                         }
1716
1717                 }
1718         }
1719
1720         /**
1721          * @param sddNamespace
1722          * @param root
1723          * @param elNodes
1724          */
1725         private void handleCharacterNodes(Namespace sddNamespace, FeatureNode root, Element elNodes) {
1726                 List<Element> listNodes = elNodes.getChildren("Node", sddNamespace);
1727                 if (listNodes != null) {
1728                         for (Element elNode : listNodes){
1729                                 String idN = elNode.getAttributeValue("id");
1730                                 FeatureNode fn = null;
1731                                 Feature dc = null;
1732                                 if (idN!=null) {
1733                                         // DescriptiveConcepts are used as nodes in CharacterTrees
1734                                         Element elDescriptiveConcept = elNode.getChild("DescriptiveConcept", sddNamespace);
1735                                         if (elDescriptiveConcept != null){
1736                                                 String refDC = elDescriptiveConcept.getAttributeValue("ref");
1737                                                 dc = features.get(refDC);
1738                                                 fn = FeatureNode.NewInstance(dc);
1739                                         }
1740                                         if (fn==null){
1741                                                 fn = FeatureNode.NewInstance();
1742                                         }
1743                                         Element elParent = elNode.getChild("Parent", sddNamespace);
1744                                         // in SDD links between Nodes are referenced by the <Parent> tag
1745                                         if (elParent!=null){
1746                                                 String refP = elParent.getAttributeValue("ref");
1747                                                 if (refP!=null) {
1748                                                         FeatureNode parent = featureNodes.get(refP);
1749                                                         if (parent==null){
1750                                                                 root.addChild(fn); // if no parent found or the reference is broken, add the node to the root of the tree
1751                                                         }
1752                                                         else {
1753                                                                 parent.addChild(fn);
1754                                                         }
1755                                                 }
1756                                         }
1757                                         else {
1758                                                 root.addChild(fn); // if no parent found or the reference is broken, add the node to the root of the tree
1759                                         }
1760                                 }
1761                                 featureNodes.put(idN, fn);
1762                         }
1763                 }
1764
1765                 // Leaves of CharacterTrees in SDD are always CharNodes (referring to Characters)
1766                 List<Element> listCharNodes = elNodes.getChildren("CharNode", sddNamespace);
1767                 if (listCharNodes != null) {
1768                         for (Element elCharNode : listCharNodes){
1769                                 Element elParent = elCharNode.getChild("Parent", sddNamespace);
1770                                 Element elCharacter = elCharNode.getChild("Character", sddNamespace);
1771                                 FeatureNode fn = FeatureNode.NewInstance();
1772                                 if (elParent!=null){
1773                                         String refP = elParent.getAttributeValue("ref");
1774                                         if ((refP!=null)&&(!refP.equals(""))) {
1775                                         FeatureNode parent = featureNodes.get(refP);
1776                                                 if (parent==null){
1777                                                 parent = root; // if no parent found or the reference is broken, add the node to the root of the tree
1778                                                 }
1779                                                 parent.addChild(fn);
1780                                         }
1781                                 }
1782                                 String refC = elCharacter.getAttributeValue("ref");
1783                                 if ((refC!=null)&&(!refC.equals(""))){
1784                                         Feature character = features.get(refC);
1785                                         fn.setFeature(character);
1786                                         featureNodes.put(refC, fn);
1787                                 }
1788                         }
1789                 }
1790         }
1791
1792         // imports the <TaxonHierarchies> block
1793         protected void importTaxonHierarchies(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig, boolean success){
1794
1795                 logger.info("start TaxonHierarchies ...");
1796                 Element elTaxonHierarchies = elDataset.getChild("TaxonHierarchies",sddNamespace);
1797
1798                 if (elTaxonHierarchies != null) {
1799                         List<Element> listTaxonHierarchies = elTaxonHierarchies.getChildren("TaxonHierarchy", sddNamespace);
1800                         int j = 0;
1801                         for (Element elTaxonHierarchy : listTaxonHierarchies){
1802                                 try {
1803                                         Element elRepresentation = elTaxonHierarchy.getChild("Representation",sddNamespace);
1804                                         String label = (String)ImportHelper.getXmlInputValue(elRepresentation,"Label",sddNamespace);
1805                                                 TaxonomicTree taxonomicTree =  TaxonomicTree.NewInstance(label);
1806                                                 importRepresentation(elTaxonHierarchy, sddNamespace, taxonomicTree, "", sddConfig);
1807
1808                                                 Set<TaxonNode> root = taxonomicTree.getChildNodes();
1809                                                 Element elNodes = elTaxonHierarchy.getChild("Nodes", sddNamespace); // There can be only one <Nodes> block for TaxonHierarchies
1810                                                 List<Element> listNodes = elNodes.getChildren("Node", sddNamespace);
1811
1812                                                 for (Element elNode : listNodes){
1813                                                         String idN = elNode.getAttributeValue("id");
1814                                                         TaxonNameBase tnb = null;
1815                                                         if (!idN.equals("")) {
1816                                                                 Element elTaxonName = elNode.getChild("TaxonName", sddNamespace);
1817                                                                 String refTN = elTaxonName.getAttributeValue("ref");
1818                                                                 tnb = taxonNameBases.get(refTN);
1819                                                                 Taxon taxon = (Taxon) tnb.getTaxa().iterator().next() ;
1820                                                                 Element elParent = elNode.getChild("Parent", sddNamespace);
1821                                                                 if (elParent!=null){
1822                                                                         String refP = elParent.getAttributeValue("ref");
1823                                                                         if (!refP.equals("")) {
1824                                                                                 TaxonNode parent = taxonNodes.get(refP);
1825                                                                                 TaxonNode child = parent.addChildTaxon(taxon, sec, "", Synonym.NewInstance(tnb, sec));
1826                                                                                 taxonNodes.put(idN,child);
1827                                                                         }
1828                                                                 }
1829                                                                 else {
1830                                                                         TaxonNode tn = taxonomicTree.addChildTaxon(taxon, sec, "", Synonym.NewInstance(tnb, sec)); // if no parent found or the reference is broken, add the node to the root of the tree
1831                                                                         taxonNodes.put(idN,tn);
1832                                                                 }
1833                                                         }
1834                                                 }
1835
1836                                                 taxonomicTrees.add(taxonomicTree);
1837                                         }
1838
1839                                 catch (Exception e) {
1840                                         //FIXME
1841                                         logger.warn("Import of Taxon Hierarchy " + j + " failed.");
1842                                         success = false;
1843                                 }
1844
1845                                 if ((++j % modCount) == 0){ logger.info("TaxonHierarchies handled: " + j);}
1846
1847                         }
1848
1849                 }
1850         }
1851
1852
1853         // imports the <GeographicAreas> block
1854         protected void importGeographicAreas(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig) {
1855                 Element elGeographicAreas = elDataset.getChild("GeographicAreas",sddNamespace);
1856                 if (elGeographicAreas != null) {
1857                         List<Element> listGeographicAreas = elGeographicAreas.getChildren("GeographicArea", sddNamespace);
1858                         int j = 0;
1859
1860                         for (Element elGeographicArea : listGeographicAreas){
1861
1862                                 String id = elGeographicArea.getAttributeValue("id");
1863                                 NamedArea na = new NamedArea();
1864                                 importRepresentation(elGeographicArea, sddNamespace, na, id, sddConfig);
1865                                 namedAreas.put(id,na);
1866                                                                 }
1867                         if ((++j % modCount) == 0){ logger.info("GeographicAreas handled: " + j);}
1868                                                         }
1869                                                         }
1870 }