cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/sdd/in/SDDImport.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.sdd.in;
  11
  12 import java.io.File;
  13 import java.net.MalformedURLException;
  14 import java.net.URL;
  15 import java.text.SimpleDateFormat;
  16 import java.util.ArrayList;
  17 import java.util.Date;
  18 import java.util.HashMap;
  19 import java.util.HashSet;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Set;
  23
  24 import org.apache.commons.lang.StringUtils;
  25 import org.apache.log4j.Logger;
  26 import org.jdom.Element;
  27 import org.jdom.Namespace;
  28 import org.joda.time.DateTime;
  29 import org.springframework.stereotype.Component;
  30 import org.springframework.transaction.TransactionStatus;
  31
  32 import eu.etaxonomy.cdm.api.service.IDescriptionService;
  33 import eu.etaxonomy.cdm.common.mediaMetaData.ImageMetaData;
  34 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  35 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  36 import eu.etaxonomy.cdm.io.common.ICdmImport;
  37 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  38 import eu.etaxonomy.cdm.io.common.ImportHelper;
  39 import eu.etaxonomy.cdm.io.sdd.SDDTransformer;
  40 import eu.etaxonomy.cdm.model.agent.Person;
  41 import eu.etaxonomy.cdm.model.agent.Team;
  42 import eu.etaxonomy.cdm.model.common.Annotation;
  43 import eu.etaxonomy.cdm.model.common.AnnotationType;
  44 import eu.etaxonomy.cdm.model.common.CdmBase;
  45 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  46 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  47 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  48 import eu.etaxonomy.cdm.model.common.Language;
  49 import eu.etaxonomy.cdm.model.common.LanguageString;
  50 import eu.etaxonomy.cdm.model.common.Marker;
  51 import eu.etaxonomy.cdm.model.common.MarkerType;
  52 import eu.etaxonomy.cdm.model.common.Representation;
  53 import eu.etaxonomy.cdm.model.common.TermBase;
  54 import eu.etaxonomy.cdm.model.common.TermVocabulary;
  55 import eu.etaxonomy.cdm.model.common.VersionableEntity;
  56 import eu.etaxonomy.cdm.model.description.CategoricalData;
  57 import eu.etaxonomy.cdm.model.description.Feature;
  58 import eu.etaxonomy.cdm.model.description.FeatureNode;
  59 import eu.etaxonomy.cdm.model.description.FeatureTree;
  60 import eu.etaxonomy.cdm.model.description.MeasurementUnit;
  61 import eu.etaxonomy.cdm.model.description.Modifier;
  62 import eu.etaxonomy.cdm.model.description.QuantitativeData;
  63 import eu.etaxonomy.cdm.model.description.State;
  64 import eu.etaxonomy.cdm.model.description.StateData;
  65 import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
  66 import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
  67 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  68 import eu.etaxonomy.cdm.model.description.TextData;
  69 import eu.etaxonomy.cdm.model.location.NamedArea;
  70 import eu.etaxonomy.cdm.model.media.IdentifiableMediaEntity;
  71 import eu.etaxonomy.cdm.model.media.ImageFile;
  72 import eu.etaxonomy.cdm.model.media.Media;
  73 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
  74 import eu.etaxonomy.cdm.model.media.MediaRepresentationPart;
  75 import eu.etaxonomy.cdm.model.media.Rights;
  76 import eu.etaxonomy.cdm.model.name.NonViralName;
  77 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  78 import eu.etaxonomy.cdm.model.occurrence.Specimen;
  79 import eu.etaxonomy.cdm.model.reference.ReferenceBase;
  80 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  81 import eu.etaxonomy.cdm.model.taxon.Synonym;
  82 import eu.etaxonomy.cdm.model.taxon.Taxon;
  83 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  84 import eu.etaxonomy.cdm.model.taxon.TaxonomicTree;
  85
  86 /**
  87  * @author h.fradin
  88  * @created 24.10.2008
  89  * @version 1.0
  90  */
  91 @Component("sddImport")
  92 public class SDDImport extends CdmImportBase<SDDImportConfigurator, SDDImportState> implements ICdmImport<SDDImportConfigurator, SDDImportState> {
  93         private static final Logger logger = Logger.getLogger(SDDImport.class);
  94
  95         private static int modCount = 1000;
  96
  97         private Map<String,Person> authors = new HashMap<String,Person>();
  98         private Map<String,String> citations = new HashMap<String,String>();
  99         private Map<String,String> defaultUnitPrefixes = new HashMap<String,String>();
 100         private Map<String,Person> editors = new HashMap<String,Person>();
 101         private Map<String,FeatureNode> featureNodes = new HashMap<String,FeatureNode>();
 102         private Map<String,Feature> features = new HashMap<String,Feature>();
 103         private Map<String,String> locations = new HashMap<String,String>();
 104         private Map<String,List<CdmBase>> mediaObject_ListCdmBase = new HashMap<String,List<CdmBase>>();
 105         private Map<String,String> mediaObject_Role = new HashMap<String,String>();
 106         private Map<String,ReferenceBase> publications = new HashMap<String,ReferenceBase>();
 107         private Map<String,State> states = new HashMap<String,State>();
 108         private Map<String,TaxonDescription> taxonDescriptions = new HashMap<String,TaxonDescription>();
 109         private Map<String,NonViralName> taxonNameBases = new HashMap<String,NonViralName>();
 110         private Map<String,MeasurementUnit> units = new HashMap<String,MeasurementUnit>();
 111         private Map<String,TaxonNode> taxonNodes = new HashMap<String,TaxonNode>();
 112         private Map<String,NamedArea> namedAreas = new HashMap<String,NamedArea>();
 113         private Map<String,Specimen> specimens = new HashMap<String,Specimen>();
 114         private Map<String,Modifier> modifiers = new HashMap<String,Modifier>();
 115
 116         private Set<MarkerType> markerTypes = new HashSet<MarkerType>();
 117         private Set<TermVocabulary> vocabularies = new HashSet<TermVocabulary>();
 118
 119         private Set<Feature> descriptiveConcepts = new HashSet<Feature>();
 120         private Set<AnnotationType> annotationTypes = new HashSet<AnnotationType>();
 121         private Set<Feature> featureSet = new HashSet<Feature>();
 122         private ReferenceBase sec = ReferenceFactory.newDatabase();
 123         private ReferenceBase sourceReference = null;
 124
 125         private Language datasetLanguage = null;
 126
 127         private Namespace xmlNamespace = Namespace.getNamespace("xml","http://www.w3.org/XML/1998/namespace");
 128
 129         private String generatorName = "";
 130         private String generatorVersion = "";
 131
 132         private Set<StatisticalMeasure> statisticalMeasures = new HashSet<StatisticalMeasure>();
 133         private Set<VersionableEntity> featureData = new HashSet<VersionableEntity>();
 134         private Set<FeatureTree> featureTrees = new HashSet<FeatureTree>();
 135         private Set<TaxonomicTree> taxonomicTrees = new HashSet<TaxonomicTree>();
 136
 137         private Rights copyright = null;
 138
 139         private int taxonNamesCount = 0; //XIM ajout
 140
 141         public SDDImport(){
 142                 super();
 143         }
 144
 145         @Override
 146         public boolean doCheck(SDDImportState state){
 147                 boolean result = true;
 148                 logger.warn("No check implemented for SDD");
 149                 return result;
 150         }
 151
 152         //      @Override
 153         //      public boolean doInvoke(IImportConfigurator config, Map<String, MapWrapper<? extends CdmBase>> stores){
 154         @Override
 155         public boolean doInvoke(SDDImportState state){
 156                 boolean success = true;
 157
 158                 TransactionStatus ts = startTransaction();
 159                 SDDImportConfigurator sddConfig = state.getConfig();
 160
 161                 logger.info("start Datasets ...");
 162                 // <Datasets xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://rs.tdwg.org/UBIF/2006/" xsi:schemaLocation="http://rs.tdwg.org/UBIF/2006/ ../SDD.xsd">
 163                 Element root = sddConfig.getSourceRoot();
 164                 Namespace sddNamespace = sddConfig.getSddNamespace();
 165
 166                 logger.info("start TechnicalMetadata ...");
 167                 // <TechnicalMetadata created="2006-04-20T10:00:00">
 168                 importTechnicalMetadata(root, sddNamespace, sddConfig);
 169                 List<Element> elDatasets = root.getChildren("Dataset",sddNamespace);
 170                 int i = 0;
 171
 172                 //for each Dataset
 173                 logger.info("start Dataset ...");
 174                 for (Element elDataset : elDatasets){
 175                         success &= importDataset(elDataset, sddNamespace, state);
 176                         if ((++i % modCount) == 0){ logger.info("Datasets handled: " + i);}
 177                         logger.info(i + " Datasets handled");
 178                 }
 179                 commitTransaction(ts);
 180                 return success;
 181         }
 182
 183         /* (non-Javadoc)
 184          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 185          */
 186         protected boolean isIgnore(SDDImportState state){
 187                 return false;
 188         }
 189
 190
 191         // associates the reference of a media object in SDD with a CdmBase Object
 192         protected void associateImageWithCdmBase(String refMO, CdmBase cb){
 193                 if ((refMO != null) && (cb!=null)) {
 194                         if (! refMO.equals("")) {
 195                                 if (! mediaObject_ListCdmBase.containsKey(refMO)) {
 196                                         List<CdmBase> lcb = new ArrayList<CdmBase>();
 197                                         lcb.add(cb);
 198                                         mediaObject_ListCdmBase.put(refMO,lcb);
 199                                 } else {
 200                                         List<CdmBase> lcb = mediaObject_ListCdmBase.get(refMO);
 201                                         lcb.add(cb);
 202                                         mediaObject_ListCdmBase.put(refMO,lcb);
 203                                 }
 204                         }
 205                 }
 206         }
 207
 208         // imports information about the Dataset
 209         protected void importDatasetRepresentation(Element parent, Namespace sddNamespace){
 210                 logger.info("start Representation ...");
 211                 /* <Representation>
 212                         <Label>The Genus Viola</Label>
 213                         <Detail>This is an example for a very simple SDD file, representing a single description with categorical, quantitative, and text character. Compare also the "Fragment*" examples, which contain more complex examples in the form of document fragments. Intended for version="SDD 1.1".</Detail>
 214                </Representation>
 215                  */
 216                 Element elRepresentation = parent.getChild("Representation",sddNamespace);
 217                 String label = (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace);
 218                 String detail = (String)ImportHelper.getXmlInputValue(elRepresentation, "Detail",sddNamespace);
 219
 220                 sec.setTitleCache(label, true);
 221
 222                 if (detail != null) {
 223                         Annotation annotation = Annotation.NewInstance(detail, datasetLanguage);
 224                         annotation.setAnnotationType(AnnotationType.EDITORIAL());
 225                         sec.addAnnotation(annotation);
 226                 }
 227
 228                 List<Element> listMediaObjects = elRepresentation.getChildren("MediaObject",sddNamespace);
 229
 230                 for (Element elMediaObject : listMediaObjects) {
 231                         String ref = null;
 232                         String role = null;
 233                         if (elMediaObject != null) {
 234                                 ref = elMediaObject.getAttributeValue("ref");
 235                                 role = elMediaObject.getAttributeValue("role");
 236                         }
 237                         if (ref != null) {
 238                                 if (!ref.equals("")) {
 239                                         this.associateImageWithCdmBase(ref,sourceReference);
 240                                         this.associateImageWithCdmBase(ref,sec);
 241                                         mediaObject_Role.put(ref,role);
 242                                 }
 243                         }
 244                 }
 245         }
 246
 247         // imports the representation (label, detail, lang) of a particular SDD element
 248         protected void importRepresentation(Element parent, Namespace sddNamespace, VersionableEntity ve, String id, IImportConfigurator config){
 249                 Element elRepresentation = parent.getChild("Representation",sddNamespace);
 250
 251                 Map<Language,List<String>> langLabDet = new HashMap<Language,List<String>>();
 252
 253                 handleRepresentationLabels(sddNamespace, elRepresentation, langLabDet);
 254                 handleRepresentationDetails(sddNamespace, elRepresentation, langLabDet);
 255
 256                 if (ve instanceof TermBase) {
 257                         makeRepresentationForTerms((TermBase)ve, langLabDet);
 258                 }else if (ve instanceof Media) {
 259                         makeRepresentationForMedia((Media)ve, langLabDet);
 260                 }else if (ve instanceof IdentifiableEntity<?>) {
 261                         IdentifiableEntity<?> ie = (IdentifiableEntity<?>)ve;
 262                         makeRepresentationForIdentifiableEntity(sddNamespace, ie, elRepresentation, langLabDet);
 263                         if (ve instanceof IdentifiableMediaEntity<?>){
 264                                 makeRepresentationForIdentifiableMediaEntity(parent, sddNamespace, (IdentifiableMediaEntity<?>)ve);
 265                         }
 266                 }
 267
 268                 makeRepresentationMediaObjects(sddNamespace, ve, elRepresentation);
 269
 270         }
 271
 272
 273         /**
 274          * Handles the "Detail" children of representations. Adds the result to the langLabDet.
 275          * @param sddNamespace
 276          * @param elRepresentation
 277          * @param langLabDet
 278          */
 279         private void handleRepresentationDetails(Namespace sddNamespace,
 280                         Element elRepresentation, Map<Language, List<String>> langLabDet) {
 281                 List<Element> listDetails = elRepresentation.getChildren("Detail",sddNamespace);
 282                 for (Element elDetail : listDetails){
 283                         Language language = getLanguage(elDetail);
 284                         String role = elDetail.getAttributeValue("role");
 285                         String detail = elDetail.getText();
 286                         List<String> labDet = langLabDet.get(language);
 287                         labDet.add(detail);
 288                         labDet.add(role);
 289                         langLabDet.put(language, labDet);
 290                 }
 291         }
 292
 293         /**
 294          * Handles the "Label" children of representations. Adds the result to the langLabDet.
 295          * @param sddNamespace
 296          * @param elRepresentation
 297          * @param langLabDet
 298          */
 299         private void handleRepresentationLabels(Namespace sddNamespace,
 300                                 Element elRepresentation, Map<Language, List<String>> langLabDet) {
 301                 // <Label xml:lang="la">Viola hederacea Labill.</Label>
 302                 List<Element> listLabels = elRepresentation.getChildren("Label",sddNamespace);
 303                 for (Element elLabel : listLabels){
 304                         Language language = getLanguage(elLabel);
 305                         String label = elLabel.getText();
 306                         List<String> labDet = new ArrayList<String>(3);
 307                         labDet.add(label);
 308                         langLabDet.put(language, labDet);
 309                 }
 310         }
 311
 312         /**
 313          *
 314          * @param ve
 315          * @param langLabDet
 316          */
 317         private void makeRepresentationForMedia(Media m, Map<Language, List<String>> langLabDet) {
 318                 for (Language lang : langLabDet.keySet()){
 319                         List<String> labDet = langLabDet.get(lang);
 320                         if (labDet.get(0) != null){
 321                                 m.addTitle(LanguageString.NewInstance(labDet.get(0), lang));
 322                         }
 323                         if (labDet.size()>1) {
 324                                 m.addDescription(labDet.get(1), lang);
 325                         }
 326                 }
 327         }
 328
 329         /**
 330          * Handles representations for terms. Adds one representation per language in langLabDet.
 331          *
 332          * @param ve
 333          * @param langLabDet
 334          */
 335         private void makeRepresentationForTerms(TermBase tb, Map<Language, List<String>> langLabDet) {
 336                         for (Language lang : langLabDet.keySet()){
 337                                 List<String> labDet = langLabDet.get(lang);
 338                                 if (labDet.size()>0){
 339                                         if (labDet.size()>1) {
 340                                                 tb.addRepresentation(Representation.NewInstance(labDet.get(1), labDet.get(0), labDet.get(0), lang));
 341                                         } else {
 342                                                 tb.addRepresentation(Representation.NewInstance(labDet.get(0), labDet.get(0), labDet.get(0), lang));
 343                                         }
 344                                 }
 345                         }
 346         }
 347
 348
 349         /**
 350          * Handles the "MediaObject" children of representations.
 351          * @param sddNamespace
 352          * @param ve
 353          * @param elRepresentation
 354          */
 355         private void makeRepresentationMediaObjects(Namespace sddNamespace,
 356                         VersionableEntity ve, Element elRepresentation) {
 357                 List <Element> listMediaObjects = elRepresentation.getChildren("MediaObject", sddNamespace);
 358                 for (Element elMediaObject : listMediaObjects) {
 359                         String ref = null;
 360                         //TODO
 361                         String role = null;
 362                         if (elMediaObject != null) {
 363                                 ref = elMediaObject.getAttributeValue("ref");
 364                                 role = elMediaObject.getAttributeValue("role");
 365                         }
 366                         if (StringUtils.isNotBlank(ref)) {
 367                                 if (ve instanceof TaxonDescription) {
 368                                         TaxonDescription td = (TaxonDescription) ve;
 369                                         //TODO: ensure that all images are imported
 370                                         if (td.getDescriptionSources().size() > 0) {
 371                                                 this.associateImageWithCdmBase(ref,(ReferenceBase) td.getDescriptionSources().toArray()[0]);
 372                                         } else {
 373                                                 ReferenceBase descriptionSource = ReferenceFactory.newGeneric();
 374                                                 td.addDescriptionSource(descriptionSource);
 375                                                 this.associateImageWithCdmBase(ref,descriptionSource);
 376                                         }
 377                                 } else {
 378                                         this.associateImageWithCdmBase(ref,ve);
 379                                 }
 380                         }
 381                 }
 382         }
 383
 384         /**
 385          * Handles the "Links" element
 386          * @param parent
 387          * @param sddNamespace
 388          * @param ve
 389          */
 390         private void makeRepresentationForIdentifiableMediaEntity(Element parent,
 391                         Namespace sddNamespace, IdentifiableMediaEntity ime) {
 392                 Element elLinks = parent.getChild("Links",sddNamespace);
 393
 394                 if (elLinks != null) {
 395
 396                         //  <Link rel="Alternate" href="http://www.diversitycampus.net/people/hagedorn"/>
 397                         List<Element> listLinks = elLinks.getChildren("Link", sddNamespace);
 398                         Media link = Media.NewInstance();
 399                         MediaRepresentation mr = MediaRepresentation.NewInstance();
 400                         int k = 0;
 401                         //for each Link
 402                         for (Element elLink : listLinks){
 403
 404                                 try {
 405                                         //TODO
 406                                         String rel = elLink.getAttributeValue("rel");
 407                                         String href = elLink.getAttributeValue("href");
 408
 409                                         mr.addRepresentationPart(MediaRepresentationPart.NewInstance(href, null));
 410                                         link.addRepresentation(mr);
 411                                         ime.addMedia(link);
 412
 413                                 } catch (Exception e) {
 414                                         //FIXME
 415                                         logger.warn("Import of Link " + k + " failed.");
 416                                 }
 417
 418                                 if ((++k % modCount) == 0){ logger.info("Links handled: " + k);}
 419
 420                         }
 421                 }
 422         }
 423
 424         /**
 425          * @param sddNamespace
 426          * @param ve
 427          * @param elRepresentation
 428          * @param langLabDet
 429          * @return
 430          */
 431         private void makeRepresentationForIdentifiableEntity(Namespace sddNamespace, IdentifiableEntity<?> ie,
 432                                         Element elRepresentation, Map<Language, List<String>> langLabDet) {
 433                 List<String> labDet = null;
 434
 435                 if (ie instanceof TaxonNameBase) {
 436                         if (langLabDet.keySet().contains(getTermService().getLanguageByIso("la"))) {
 437                                 labDet = langLabDet.get(getTermService().getLanguageByIso("la"));
 438                         } else if (langLabDet.keySet().contains(datasetLanguage)) {
 439                                 labDet = langLabDet.get(datasetLanguage);
 440                                 logger.info("TaxonName " + (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace) + " is not specified as a latin name.");
 441                         } else {
 442                                 labDet = langLabDet.get(langLabDet.keySet().iterator().next());
 443                                 logger.info("TaxonName " + (String)ImportHelper.getXmlInputValue(elRepresentation, "Label",sddNamespace) + " is not specified as a latin name.");
 444                         }
 445                 } else {
 446                         labDet = langLabDet.get(langLabDet.keySet().iterator().next());
 447                 }
 448
 449                 //FIXME labDet is != null only for TaxonNameBase
 450                 ie.setTitleCache(labDet.get(0), true);
 451
 452                 if (labDet.size()>1) {
 453                         Annotation annotation = null;
 454                         if (labDet.get(1) != null) {
 455                                 if (labDet.get(2) != null) {
 456                                         annotation = Annotation.NewInstance(labDet.get(2) + " - " + labDet.get(1), datasetLanguage);
 457                                 } else {
 458                                         annotation = Annotation.NewInstance(labDet.get(1), datasetLanguage);
 459                                 }
 460                         }
 461                         ie.addAnnotation(annotation);
 462                 }
 463                 return;
 464         }
 465
 466         /**
 467          * @param elLabel
 468          * @return
 469          */
 470         private Language getLanguage(Element elLanguage) {
 471                 String lang = elLanguage.getAttributeValue("lang",xmlNamespace);
 472                 Language language = null;
 473                 if (StringUtils.isNotBlank(lang)) {
 474                         language = getTermService().getLanguageByIso(lang.substring(0, 2));
 475                 } else {
 476                         language = datasetLanguage;
 477                 }
 478                 return language;
 479         }
 480
 481
 482         // imports the representation (label, detail, lang) of a particular SDD element
 483         protected void importTechnicalMetadata(Element root, Namespace sddNamespace, SDDImportConfigurator sddConfig){
 484                 Element elTechnicalMetadata = root.getChild("TechnicalMetadata", sddNamespace);
 485                 String nameCreated = elTechnicalMetadata.getAttributeValue("created");
 486                 sourceReference = sddConfig.getSourceReference();
 487
 488                 if (nameCreated != null) {
 489                         if (!nameCreated.equals("")) {
 490                                 int year = Integer.parseInt(nameCreated.substring(0,4));
 491                                 int monthOfYear = Integer.parseInt(nameCreated.substring(5,7));
 492                                 int dayOfMonth = Integer.parseInt(nameCreated.substring(8,10));
 493                                 int hourOfDay = Integer.parseInt(nameCreated.substring(11,13));
 494                                 int minuteOfHour = Integer.parseInt(nameCreated.substring(14,16));
 495                                 int secondOfMinute = Integer.parseInt(nameCreated.substring(17,19));
 496                                 DateTime created = new DateTime(year,monthOfYear,dayOfMonth,hourOfDay,minuteOfHour,secondOfMinute,0);
 497                                 sourceReference.setCreated(created);
 498                                 sec.setCreated(created);
 499                         }
 500                 }
 501
 502                 // <Generator name="n/a, handcrafted instance document" version="n/a"/>
 503                 Element elGenerator = elTechnicalMetadata.getChild("Generator", sddNamespace);
 504                 generatorName = elGenerator.getAttributeValue("name");
 505                 generatorVersion = elGenerator.getAttributeValue("version");
 506
 507                 sec.addAnnotation(Annotation.NewDefaultLanguageInstance(generatorName + " - " + generatorVersion));
 508                 sourceReference.addAnnotation(Annotation.NewDefaultLanguageInstance(generatorName + " - " + generatorVersion));
 509
 510         }
 511
 512         // imports the complete dataset information
 513         protected boolean importDataset(Element elDataset, Namespace sddNamespace, SDDImportState state){                       // <Dataset xml:lang="en-us">
 514                 boolean success = true;
 515                 SDDImportConfigurator sddConfig = state.getConfig();
 516
 517                 importDatasetLanguage(elDataset,sddConfig);
 518                 importDatasetRepresentation(elDataset, sddNamespace);
 519                 importRevisionData(elDataset, sddNamespace);
 520                 importIPRStatements(elDataset, sddNamespace, sddConfig);
 521                 importTaxonNames(elDataset, sddNamespace, sddConfig);
 522
 523                 importDescriptiveConcepts(elDataset, sddNamespace, sddConfig);
 524                 success &= importCharacters(elDataset, sddNamespace, sddConfig);
 525                 importCharacterTrees(elDataset, sddNamespace, sddConfig, success);
 526
 527                 MarkerType editorMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerEditor, "editor", "Editor", "edt");
 528                 MarkerType geographicAreaMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerSDDGeographicArea, "SDDGeographicArea", "SDDGeographicArea", "ga");
 529                 MarkerType descriptiveConceptMarkerType = getMarkerType(state, SDDTransformer.uuidMarkerDescriptiveConcept, "DescriptiveConcept", "Descriptive Concept", "DC");
 530                 markerTypes.add(editorMarkerType);
 531                 markerTypes.add(geographicAreaMarkerType);
 532                 markerTypes.add(descriptiveConceptMarkerType);
 533
 534                 //saving of all imported data into the CDM db
 535                 saveVocabularies();
 536                 saveFeatures();
 537                 saveModifiers();
 538                 saveStates();
 539                 saveMarkerType();
 540                 saveAreas(geographicAreaMarkerType);
 541                 saveUnits();
 542                 saveStatisticalMeasure();
 543                 saveAnnotationType();
 544
 545                 success &= importCodedDescriptions(elDataset, sddNamespace, sddConfig);
 546                 importAgents(elDataset, sddNamespace, sddConfig, success);
 547                 importPublications(elDataset, sddNamespace, sddConfig, success);
 548                 importMediaObjects(elDataset, sddNamespace, sddConfig, success);
 549                 importTaxonHierarchies(elDataset, sddNamespace, sddConfig, success);
 550                 importGeographicAreas(elDataset, sddNamespace, sddConfig);
 551                 importSpecimens(elDataset,sddNamespace, sddConfig);
 552
 553
 554                 if ((authors != null)||(editors != null)) {
 555                         Team team = Team.NewInstance();
 556                         if (authors != null) {
 557                                 for (Person author : authors.values()){
 558                                         team.addTeamMember(author);
 559                                 }
 560                         }
 561                         if (editors != null) {
 562                                 Marker marker = Marker.NewInstance();
 563                                 marker.setMarkerType(editorMarkerType);
 564                                 for (Person editor : editors.values()){
 565                                         Person edit = editor;
 566                                         edit.addMarker(marker);
 567                                         team.addTeamMember(edit);
 568                                 }
 569                         }
 570                         sec.setAuthorTeam(team);
 571                         sourceReference.setAuthorTeam(team);
 572                 }
 573
 574                 if (copyright != null) {
 575                         sourceReference.addRights(copyright);
 576                         sec.addRights(copyright);
 577                 }
 578
 579                 // Returns a CdmApplicationController created by the values of this configuration.
 580                 IDescriptionService descriptionService = getDescriptionService();
 581
 582                 for (TaxonDescription taxonDescription : taxonDescriptions.values()){
 583                         // Persists a Description
 584                         descriptionService.save(taxonDescription);
 585                 }
 586
 587                 for (String ref : taxonDescriptions.keySet()){
 588                         TaxonDescription td = taxonDescriptions.get(ref);
 589                         if (citations.containsKey(ref)) {
 590                                 ReferenceBase publication = publications.get(citations.get(ref));
 591                                 if (locations.containsKey(ref)) {
 592                                         Annotation location = Annotation.NewInstance(locations.get(ref), datasetLanguage);
 593                                         AnnotationType annotationType = AnnotationType.NewInstance("", "location", "");
 594                                         annotationTypes.add(annotationType);
 595                                         location.setAnnotationType(annotationType);
 596                                         (publication).addAnnotation(location);
 597                                 }
 598                                 td.addDescriptionSource(publication);
 599                         }
 600                 }
 601                 logger.info("end makeTaxonDescriptions ...");
 602
 603
 604 //              for (Iterator<TermVocabulary<Modifier>> k = termVocabularyStates.iterator() ; k.hasNext() ;){
 605 //                      TermVocabulary<Modifier> termVocabulary = k.next();
 606 //                      getVocabularyService().save(termVocabulary); //XIM
 607 //              }
 608
 609
 610                 //sddConfig.setSourceReference(sourceReference);
 611
 612
 613                 if (descriptiveConcepts != null) {
 614                         for (Feature feature : descriptiveConcepts) {
 615                                 Marker marker = Marker.NewInstance();
 616                                 marker.setMarkerType(descriptiveConceptMarkerType);
 617                                 feature.addMarker(marker);
 618                         }
 619                 }
 620                 saveFeatures();
 621
 622                 /*Marker markerd = Marker.NewInstance();
 623                 markerd.setMarkerType(descriptiveConceptMarker);
 624                 Feature fiture = Feature.NewInstance("Fitoure","Fitoure","Fitoure");
 625                 fiture.addMarker(markerd);
 626                 TermVocabulary<Modifier> termVocabularyState = new TermVocabulary<Modifier>("test","test","test","test");
 627                 Modifier modif = new Modifier("zoub","zab","zib");
 628                 termVocabularyState.addTerm(modif);
 629                 getVocabularyService().save(termVocabularyState);
 630                 fiture.addRecommendedModifierEnumeration(termVocabularyState);
 631                 termService.save(modif);
 632                 termService.save(fiture);*/
 633
 634                 //XIMtermService.save(editorMarkerType);
 635
 636                 //XIMtermService.save(geographicAreaMarkerType);
 637
 638                 // referenceService.saveReference(sourceReference);
 639                 for (ReferenceBase publication : publications.values()){
 640                         getReferenceService().save(publication);
 641                 }
 642
 643                 for (FeatureTree featureTree : featureTrees) {
 644                         getFeatureTreeService().save(featureTree);
 645                 }
 646                 for (TaxonomicTree taxonomicTree : taxonomicTrees) {
 647                         getTaxonTreeService().save(taxonomicTree);
 648                 }
 649                 for (Specimen specimen : specimens.values()) {
 650                         getOccurrenceService().save(specimen);
 651                 }
 652                 logger.info("end of persistence ...");
 653
 654                 return success;
 655         }
 656
 657         /**
 658          *
 659          */
 660         private void saveVocabularies() {
 661                 for (TermVocabulary vocabulary : vocabularies ){
 662                         getVocabularyService().save(vocabulary);
 663                 }
 664
 665         }
 666
 667         private void saveAnnotationType() {
 668                 for (AnnotationType annotationType: annotationTypes){
 669                         getTermService().save(annotationType);
 670                 }
 671         }
 672
 673         private void saveStatisticalMeasure() {
 674                 for (StatisticalMeasure sm : statisticalMeasures){
 675                         getTermService().save(sm);
 676                 }
 677         }
 678
 679         private void saveUnits() {
 680                 if (units != null) {
 681                         for (MeasurementUnit unit : units.values()){
 682                                 if (unit != null) {
 683                                         getTermService().save(unit);
 684                                 }
 685                         }
 686                 }
 687         }
 688
 689         private void saveAreas(MarkerType geographicAreaMarkerType) {
 690                 for (NamedArea area : namedAreas.values() ){
 691                         Marker marker = Marker.NewInstance();
 692                         marker.setMarkerType(geographicAreaMarkerType);
 693                         area.addMarker(marker);
 694                         getTermService().save(area);
 695                 }
 696         }
 697
 698         private void saveStates() {
 699                 for (State state : states.values() ){
 700                         getTermService().save(state);
 701                 }
 702         }
 703
 704         private void saveMarkerType() {
 705                 for (MarkerType markerType : markerTypes){
 706                         getTermService().save(markerType);
 707                 }
 708         }
 709
 710         private void saveModifiers() {
 711                 for (Modifier modifier : modifiers.values() ){
 712                         getTermService().save(modifier);
 713                 }
 714         }
 715
 716         private void saveFeatures() {
 717                 for (Feature feature : features.values() ){
 718                         getTermService().save(feature);
 719                 }
 720         }
 721
 722         // imports the default language of the dataset
 723         protected void importDatasetLanguage(Element elDataset, SDDImportConfigurator sddConfig){
 724                 String nameLang = elDataset.getAttributeValue("lang",xmlNamespace);
 725
 726                 if (!nameLang.equals("")) {
 727                         String iso = nameLang.substring(0, 2);
 728                         datasetLanguage = getTermService().getLanguageByIso(iso);
 729                 } else {
 730                         datasetLanguage = Language.DEFAULT();
 731                 }
 732                 if (datasetLanguage == null) {
 733                         datasetLanguage = Language.DEFAULT();
 734                 }
 735         }
 736
 737         // imports the specimens
 738         protected void importSpecimens(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig) {
 739                 logger.info("start Specimens ...");
 740                 /*      <Specimens>
 741                         <Specimen id="sp1">
 742                                 <Representation>
 743                                 <Label>TJM45337</Label>
 744                                 </Representation>
 745                         </Specimen>
 746                 </Specimens>
 747                  */
 748                 Element elSpecimens = elDataset.getChild("Specimens",sddNamespace);
 749                 if (elSpecimens != null){
 750                         List<Element> listSpecimens = elSpecimens.getChildren("Specimen", sddNamespace);
 751                         int j = 0;
 752                         for (Element elSpecimen : listSpecimens) {
 753                                 String id = elSpecimen.getAttributeValue("id");
 754                                 Specimen specimen = null;
 755                                 if (!id.equals("")) {
 756                                         specimen = Specimen.NewInstance();
 757                                         specimens.put(id,specimen);
 758                                         importRepresentation(elSpecimen, sddNamespace, specimen, id, sddConfig);
 759                                 }
 760                         }
 761
 762                 }
 763         }
 764
 765         // imports the revision data associated with the Dataset (authors, modifications)
 766         protected void importRevisionData(Element elDataset, Namespace sddNamespace){
 767                 // <RevisionData>
 768                 logger.info("start RevisionData ...");
 769                 Element elRevisionData = elDataset.getChild("RevisionData",sddNamespace);
 770                 if (elRevisionData != null){
 771                         // <Creators>
 772                         Element elCreators = elRevisionData.getChild("Creators",sddNamespace);
 773
 774                         // <Agent role="aut" ref="a1"/>
 775                         List<Element> listAgents = elCreators.getChildren("Agent", sddNamespace);
 776
 777                         int j = 0;
 778                         //for each Agent
 779                         for (Element elAgent : listAgents){
 780
 781                                 String role = elAgent.getAttributeValue("role");
 782                                 String ref = elAgent.getAttributeValue("ref");
 783                                 if (role.equals("aut")) {
 784                                         if(!ref.equals("")) {
 785                                                 authors.put(ref, null);
 786                                         }
 787                                 }
 788                                 if (role.equals("edt")) {
 789                                         if(!ref.equals("")) {
 790                                                 editors.put(ref, null);
 791                                         }
 792                                 }
 793                                 if ((++j % modCount) == 0){ logger.info("Agents handled: " + j);}
 794
 795                         }
 796
 797                         // <DateModified>2006-04-08T00:00:00</DateModified>
 798                         String stringDateModified = (String)ImportHelper.getXmlInputValue(elRevisionData, "DateModified",sddNamespace);
 799
 800                         if (stringDateModified != null) {
 801                                 SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss");
 802                                 Date d = null;
 803                                 try {
 804                                         d = sdf.parse(stringDateModified);
 805                                 } catch(Exception e) {
 806                                         System.err.println("Exception :");
 807                                         e.printStackTrace();
 808                                 }
 809
 810                                 DateTime updated = null;
 811                                 if (d != null) {
 812                                         updated = new DateTime(d);
 813                                         sourceReference.setUpdated(updated);
 814                                         sec.setUpdated(updated);
 815                                 }
 816                         }
 817                 }
 818         }
 819
 820         // imports ipr statements associated with a dataset
 821         protected void importIPRStatements(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig){
 822                 // <IPRStatements>
 823                 logger.info("start IPRStatements ...");
 824                 Element elIPRStatements = elDataset.getChild("IPRStatements",sddNamespace);
 825                 // <IPRStatement role="Copyright">
 826                 if (elIPRStatements != null) {
 827                         List<Element> listIPRStatements = elIPRStatements.getChildren("IPRStatement", sddNamespace);
 828                         int j = 0;
 829                         //for each IPRStatement
 830
 831                         for (Element elIPRStatement : listIPRStatements){
 832
 833                                 String role = elIPRStatement.getAttributeValue("role");
 834                                 // <Label xml:lang="en-au">(c) 2003-2006 Centre for Occasional Botany.</Label>
 835                                 Element elLabel = elIPRStatement.getChild("Label",sddNamespace);
 836                                 String lang = "";
 837                                 if (elLabel != null) {
 838                                         lang = elLabel.getAttributeValue("lang",xmlNamespace);
 839                                 }
 840                                 String label = (String)ImportHelper.getXmlInputValue(elIPRStatement, "Label",sddNamespace);
 841
 842                                 if (role.equals("Copyright")) {
 843                                         Language iprLanguage = null;
 844                                         if (lang != null) {
 845                                                 if (!lang.equals("")) {
 846                                                         iprLanguage = getTermService().getLanguageByIso(lang.substring(0, 2));
 847                                                 } else {
 848                                                         iprLanguage = datasetLanguage;
 849                                                 }
 850                                         }
 851                                         if (iprLanguage == null) {
 852                                                 iprLanguage = datasetLanguage;
 853                                         }
 854                                         copyright = Rights.NewInstance(label, iprLanguage);
 855                                 }
 856
 857                                 if (copyright != null) {
 858                                         sourceReference.addRights(copyright);
 859                                         sec.addRights(copyright);
 860                                 }
 861
 862                                 if ((++j % modCount) == 0){ logger.info("IPRStatements handled: " + j);}
 863
 864                         }
 865                 }
 866         }
 867
 868         // imports the taxon names
 869         protected void importTaxonNames(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig){
 870                 // <TaxonNames>
 871                 logger.info("start TaxonNames ...");
 872                 Element elTaxonNames = elDataset.getChild("TaxonNames",sddNamespace);
 873                 // <TaxonName id="t1" uri="urn:lsid:authority:namespace:my-own-id">
 874                 if (elTaxonNames != null) {
 875                         List<Element> listTaxonNames = elTaxonNames.getChildren("TaxonName", sddNamespace);
 876                         int j = 0;
 877                         //for each TaxonName
 878                         for (Element elTaxonName : listTaxonNames){
 879
 880                                 String id = elTaxonName.getAttributeValue("id");
 881                                 String uri = elTaxonName.getAttributeValue("uri");
 882
 883                                 NonViralName tnb = null;
 884                                 if (!id.equals("")) {
 885                                         tnb = NonViralName.NewInstance(null);
 886                                         IdentifiableSource source = null;
 887                                         if (uri != null) {
 888                                                 if (!uri.equals("")) {
 889                                                         source = IdentifiableSource.NewInstance(id, "TaxonName", ReferenceFactory.newGeneric(), uri);
 890                                                 }
 891                                         } else {
 892                                                 source = IdentifiableSource.NewInstance(id, "TaxonName");
 893                                         }
 894                                         tnb.addSource(source);
 895                                         taxonNameBases.put(id,tnb);
 896                                 }
 897
 898                                 // <Representation>
 899                                 // <Label xml:lang="la">Viola hederacea Labill.</Label>
 900                                 importRepresentation(elTaxonName, sddNamespace, tnb, id, sddConfig);
 901
 902                                 if ((++j % modCount) == 0){ logger.info("TaxonNames handled: " + j);}
 903
 904                         }
 905                 }
 906         }
 907
 908         // imports the characters (categorical, quantitative and text ; sequence characters not supported) which correspond to CDM Features
 909         protected boolean importCharacters(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig){
 910                 boolean success = true;
 911                 // <Characters>
 912                 logger.info("start Characters ...");
 913                 Element elCharacters = elDataset.getChild("Characters", sddNamespace);
 914
 915                 // <CategoricalCharacter id="c1">
 916                 if (elCharacters != null) {
 917                         success &= handleCategoricalData(sddNamespace, sddConfig, elCharacters);
 918                         success &= handleQuantitativeData(sddNamespace, sddConfig, elCharacters);
 919                         success &= handleTextCharacters(sddNamespace, sddConfig, elCharacters);
 920                 }
 921
 922                 /*for (Iterator<Feature> f = features.values().iterator() ; f.hasNext() ;){
 923                         featureSet.add(f.next()); //XIM Why this line ?
 924                 }*/
 925
 926                 return success;
 927
 928         }
 929
 930         /**
 931          * @param sddNamespace
 932          * @param sddConfig
 933          * @param success
 934          * @param elCharacters
 935          * @return
 936          */
 937         private boolean handleCategoricalData(Namespace sddNamespace, SDDImportConfigurator sddConfig, Element elCharacters) {
 938                 boolean success = true;
 939                 List<Element> elCategoricalCharacters = elCharacters.getChildren("CategoricalCharacter", sddNamespace);
 940                 int j = 0;
 941                 for (Element elCategoricalCharacter : elCategoricalCharacters){
 942                         try {
 943
 944                                 String idCC = elCategoricalCharacter.getAttributeValue("id");
 945                                 Feature categoricalCharacter = Feature.NewInstance();
 946                                 categoricalCharacter.setKindOf(Feature.DESCRIPTION());
 947                                 importRepresentation(elCategoricalCharacter, sddNamespace, categoricalCharacter, idCC, sddConfig);
 948                                 categoricalCharacter.setSupportsCategoricalData(true);
 949
 950                                 // <States>
 951                                 Element elStates = elCategoricalCharacter.getChild("States",sddNamespace);
 952
 953                                 // <StateDefinition id="s1">
 954                                 List<Element> elStateDefinitions = elStates.getChildren("StateDefinition",sddNamespace);
 955                                 TermVocabulary<State> termVocabularyState = new TermVocabulary<State>();
 956
 957                                 vocabularies.add(termVocabularyState);
 958
 959                                 int k = 0;
 960                                 //for each StateDefinition
 961                                 for (Element elStateDefinition : elStateDefinitions){
 962
 963                                         if ((++k % modCount) == 0){ logger.info("StateDefinitions handled: " + (k-1));}
 964
 965                                         String idS = elStateDefinition.getAttributeValue("id");
 966                                         State state = states.get(idS);
 967                                         if (state == null){
 968                                                 state = State.NewInstance();
 969                                         }else{
 970                                                 logger.debug("State duplicate found");
 971                                         }
 972                                         importRepresentation(elStateDefinition, sddNamespace, state, idS, sddConfig);
 973
 974                                         //StateData stateData = StateData.NewInstance();
 975                                         //stateData.setState(state);
 976                                         termVocabularyState.addTerm(state);
 977                                         states.put(idS,state);
 978                                 }
 979                                 categoricalCharacter.addSupportedCategoricalEnumeration(termVocabularyState);
 980                                 features.put(idCC, categoricalCharacter);
 981
 982                         } catch (Exception e) {
 983                                 logger.warn("Import of CategoricalCharacter " + j + " failed.");
 984                                 success = false;
 985                         }
 986
 987                         if ((++j % modCount) == 0){ logger.info("CategoricalCharacters handled: " + j);}
 988
 989                 }
 990                 return success;
 991         }
 992
 993         /**
 994          * @param sddNamespace
 995          * @param sddConfig
 996          * @param elCharacters
 997          */
 998         private boolean handleQuantitativeData(Namespace sddNamespace,  SDDImportConfigurator sddConfig, Element elCharacters) {
 999                 boolean success = true;
1000                 int j;
1001                 // <QuantitativeCharacter id="c2">
1002                 List<Element> elQuantitativeCharacters = elCharacters.getChildren("QuantitativeCharacter", sddNamespace);
1003                 j = 0;
1004                 //for each QuantitativeCharacter
1005                 for (Element elQuantitativeCharacter : elQuantitativeCharacters){
1006
1007                         try {
1008
1009                                 String idQC = elQuantitativeCharacter.getAttributeValue("id");
1010
1011                                 // <Representation>
1012                                 //  <Label>Leaf length</Label>
1013                                 // </Representation>
1014                                 Feature quantitativeCharacter = Feature.NewInstance();
1015                                 quantitativeCharacter.setKindOf(Feature.DESCRIPTION());
1016                                 importRepresentation(elQuantitativeCharacter, sddNamespace, quantitativeCharacter, idQC, sddConfig);
1017
1018                                 quantitativeCharacter.setSupportsQuantitativeData(true);
1019
1020                                 // <MeasurementUnit>
1021                                 //  <Label role="Abbrev">m</Label>
1022                                 // </MeasurementUnit>
1023                                 Element elMeasurementUnit = elQuantitativeCharacter.getChild("MeasurementUnit",sddNamespace);
1024                                 String label = "";
1025                                 String role = "";
1026                                 if (elMeasurementUnit != null) {
1027                                         Element elLabel = elMeasurementUnit.getChild("Label",sddNamespace);
1028                                         role = elLabel.getAttributeValue("role");
1029                                         label = (String)ImportHelper.getXmlInputValue(elMeasurementUnit, "Label",sddNamespace);
1030                                 }
1031
1032                                 MeasurementUnit unit = null;
1033                                 if (!label.equals("")){
1034                                         if (role != null) {
1035                                                 if (role.equals("Abbrev")){
1036                                                         unit = MeasurementUnit.NewInstance(label,label,label);
1037                                                 }
1038                                         } else {
1039                                                 unit = MeasurementUnit.NewInstance(label,label,label);
1040                                         }
1041                                 }
1042
1043                                 if (unit != null) {
1044                                         units.put(idQC, unit);
1045                                 }
1046
1047                                 //<Default>
1048                                 //  <MeasurementUnitPrefix>milli</MeasurementUnitPrefix>
1049                                 //</Default>
1050                                 Element elDefault = elQuantitativeCharacter.getChild("Default",sddNamespace);
1051                                 if (elDefault != null) {
1052                                         String measurementUnitPrefix = (String)ImportHelper.getXmlInputValue(elDefault, "MeasurementUnitPrefix",sddNamespace);
1053                                         if (! measurementUnitPrefix.equals("")){
1054                                                 defaultUnitPrefixes.put(idQC, measurementUnitPrefix);
1055                                         }
1056                                 }
1057
1058                                 features.put(idQC, quantitativeCharacter);
1059
1060                         } catch (Exception e) {
1061                                 //FIXME
1062                                 logger.warn("Import of QuantitativeCharacter " + j + " failed.");
1063                                 success = false;
1064                         }
1065
1066                         if ((++j % modCount) == 0){ logger.info("QuantitativeCharacters handled: " + j);}
1067
1068                 }
1069                 return success;
1070         }
1071
1072         private boolean handleTextCharacters(Namespace sddNamespace, SDDImportConfigurator sddConfig, Element elCharacters) {
1073                 boolean success = true;
1074                 int j;
1075                 // <TextCharacter id="c3">
1076                 List<Element> elTextCharacters = elCharacters.getChildren("TextCharacter", sddNamespace);
1077                 j = 0;
1078                 //for each TextCharacter
1079                 for (Element elTextCharacter : elTextCharacters){
1080
1081                         try {
1082
1083                                 String idTC = elTextCharacter.getAttributeValue("id");
1084
1085                                 // <Representation>
1086                                 //  <Label xml:lang="en">Leaf features not covered by other characters</Label>
1087                                 // </Representation>
1088                                 Feature textCharacter = Feature.NewInstance();
1089                                 textCharacter.setKindOf(Feature.DESCRIPTION());
1090                                 importRepresentation(elTextCharacter, sddNamespace, textCharacter, idTC, sddConfig);
1091
1092                                 textCharacter.setSupportsTextData(true);
1093
1094                                 features.put(idTC, textCharacter);
1095
1096                         } catch (Exception e) {
1097                                 //FIXME
1098                                 logger.warn("Import of TextCharacter " + j + " failed.");
1099                                 success = false;
1100                         }
1101
1102                         if ((++j % modCount) == 0){ logger.info("TextCharacters handled: " + j);}
1103
1104                 }
1105                 return success;
1106         }
1107
1108         // imports the descriptions of taxa
1109         protected boolean importCodedDescriptions(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig){
1110                 boolean success = true;
1111
1112                 // <CodedDescriptions>
1113                 logger.info("start CodedDescriptions ...");
1114                 Element elCodedDescriptions = elDataset.getChild("CodedDescriptions",sddNamespace);
1115
1116                 // <CodedDescription id="D101">
1117                 if (elCodedDescriptions != null) {
1118                         List<Element> listCodedDescriptions = elCodedDescriptions.getChildren("CodedDescription", sddNamespace);
1119                         int j = 0;
1120                         //for each CodedDescription
1121                         for (Element elCodedDescription : listCodedDescriptions){
1122                                 success &= handleCodedDescription(sddNamespace, sddConfig, elCodedDescription, j);
1123                                 if ((++j % modCount) == 0){ logger.info("CodedDescriptions handled: " + j);}
1124                         }
1125                 }
1126                 return success;
1127         }
1128
1129         /**
1130          * @param sddNamespace
1131          * @param sddConfig
1132          * @param j
1133          * @param elCodedDescription
1134          * @return
1135          */
1136         private boolean handleCodedDescription(Namespace sddNamespace, SDDImportConfigurator sddConfig, Element elCodedDescription, int j) {
1137                 boolean success = true ;
1138                 try {
1139
1140                         String idCD = elCodedDescription.getAttributeValue("id");
1141
1142                         // <Representation>
1143                         //  <Label>&lt;i&gt;Viola hederacea&lt;/i&gt; Labill. as revised by R. Morris April 8, 2006</Label>
1144                         // </Representation>
1145                         TaxonDescription taxonDescription = TaxonDescription.NewInstance();
1146                         importRepresentation(elCodedDescription, sddNamespace, taxonDescription, idCD, sddConfig);
1147
1148                         // <Scope>
1149                         //  <TaxonName ref="t1"/>
1150                         //  <Citation ref="p1" location="p. 30"/>
1151                         // </Scope>
1152                         Element elScope = elCodedDescription.getChild("Scope", sddNamespace);
1153                         Taxon taxon;
1154                         if (elScope != null) {
1155                                 taxon = handleCDScope(sddNamespace, sddConfig, idCD, elScope);
1156                         } else {//in case no taxon is linked to the description, a new one is created
1157                                 taxon = handleCDNoScope(sddNamespace, sddConfig, elCodedDescription);
1158                         }
1159
1160                         // <SummaryData>
1161                         Element elSummaryData = elCodedDescription.getChild("SummaryData",sddNamespace);
1162                         if (elSummaryData != null) {
1163                                 handleSummaryCategoricalData(sddNamespace, taxonDescription, elSummaryData);
1164                                 handleSummaryQuantitativeData(sddNamespace, taxonDescription, elSummaryData);
1165                                 handleSummaryTextData(sddNamespace, taxonDescription, elSummaryData);
1166                         }
1167
1168                         if (taxon != null) {
1169                                 taxon.addDescription(taxonDescription);
1170                         }
1171
1172                         taxonDescription.setDescriptiveSystem(featureSet);
1173
1174                         taxonDescriptions.put(idCD, taxonDescription);//FIXME
1175
1176                 } catch (Exception e) {
1177                         //FIXME
1178                         logger.warn("Import of CodedDescription " + j + " failed.", e);
1179                         success = false;
1180                 }
1181                 return success;
1182         }
1183
1184         /**
1185          * @param sddNamespace
1186          * @param sddConfig
1187          * @param elCodedDescription
1188          * @param taxon
1189          * @return
1190          */
1191         private Taxon handleCDNoScope(Namespace sddNamespace,
1192                         SDDImportConfigurator sddConfig, Element elCodedDescription     ) {
1193                 Taxon taxon = null;
1194                 NonViralName nonViralName = NonViralName.NewInstance(null);
1195                 String id = new String("" + taxonNamesCount);
1196                 IdentifiableSource source = IdentifiableSource.NewInstance(id, "TaxonName");
1197                 importRepresentation(elCodedDescription, sddNamespace, nonViralName, id, sddConfig);
1198
1199                 if(sddConfig.isDoMatchTaxa()){
1200                         taxon = getTaxonService().findBestMatchingTaxon(nonViralName.getTitleCache());
1201                 }
1202
1203                 if(taxon != null){
1204                         nonViralName = HibernateProxyHelper.deproxy(taxon.getName(), NonViralName.class);
1205 //                                                      taxonNameBases.put(id ,tnb);
1206 //                                                      taxonNamesCount++;
1207                         logger.info("using existing Taxon" + taxon.getTitleCache());
1208                 } else {
1209                         nonViralName.addSource(source);
1210                         taxonNameBases.put(id ,nonViralName);
1211                         taxonNamesCount++;
1212                         logger.info("creating new Taxon from TaxonName" + nonViralName.getTitleCache());
1213                         taxon = Taxon.NewInstance(nonViralName, sec);
1214                 }
1215                 return taxon;
1216         }
1217
1218         /**
1219          * @param sddNamespace
1220          * @param sddConfig
1221          * @param idCD
1222          * @param elScope
1223          * @param taxon
1224          * @return
1225          */
1226         private Taxon handleCDScope(Namespace sddNamespace, SDDImportConfigurator sddConfig,
1227                         String idCD, Element elScope) {
1228                 Taxon taxon = null;
1229                 Element elTaxonName = elScope.getChild("TaxonName", sddNamespace);
1230                 String ref = elTaxonName.getAttributeValue("ref");
1231                 NonViralName nonViralName = taxonNameBases.get(ref);
1232
1233                 if(sddConfig.isDoMatchTaxa()){
1234                         taxon = getTaxonService().findBestMatchingTaxon(nonViralName.getTitleCache());
1235                 }
1236
1237                 if(taxon != null){
1238                         logger.info("using existing Taxon" + taxon.getTitleCache());
1239                         if(!nonViralName.getUuid().equals(taxon.getName().getUuid())){
1240                                 logger.warn("TaxonNameBase entity of existing taxon does not match Name in list -> replacing Name in list");
1241                                 nonViralName = HibernateProxyHelper.deproxy(taxon.getName(), NonViralName.class);
1242                         }
1243                 } else {
1244                         logger.info("creating new Taxon from TaxonName" + nonViralName.getTitleCache());
1245                         taxon = Taxon.NewInstance(nonViralName, sec);
1246                 }
1247
1248                 //citation
1249                 Element elCitation = elScope.getChild("Citation",sddNamespace);
1250                 if (elCitation != null) {
1251                         String refCitation = elCitation.getAttributeValue("ref");
1252                         if (! refCitation.equals("")){
1253                                 citations.put(idCD, refCitation);
1254                         }
1255                         String location = elCitation.getAttributeValue("location");
1256                         if (! location.equals("")){
1257                                 locations.put(idCD, location);
1258                         }
1259                 }
1260                 return taxon;
1261         }
1262
1263         /**
1264          * @param sddNamespace
1265          * @param taxonDescription
1266          * @param elSummaryData
1267          */
1268         private void handleSummaryTextData(Namespace sddNamespace,
1269                         TaxonDescription taxonDescription, Element elSummaryData) {
1270                 String ref;
1271                 int k;
1272                 // <TextChar ref="c3">
1273                 List<Element> elTextChars = elSummaryData.getChildren("TextChar", sddNamespace);
1274                 k = 0;
1275                 //for each TextChar
1276                 for (Element elTextChar : elTextChars){
1277                         if ((++k % modCount) == 0){ logger.info("TextChar handled: " + (k-1));}
1278                         ref = elTextChar.getAttributeValue("ref");
1279                         Feature feature = features.get(ref);
1280                         TextData textData = TextData.NewInstance();
1281                         textData.setFeature(feature);
1282
1283                         // <Content>Free form text</Content>
1284                         String content = (String)ImportHelper.getXmlInputValue(elTextChar, "Content",sddNamespace);
1285                         textData.putText(content, datasetLanguage);
1286                         taxonDescription.addElement(textData);
1287                 }
1288         }
1289
1290         /**
1291          * @param sddNamespace
1292          * @param taxonDescription
1293          * @param elSummaryData
1294          */
1295         private void handleSummaryQuantitativeData(Namespace sddNamespace,
1296                         TaxonDescription taxonDescription, Element elSummaryData) {
1297                 String ref;
1298                 int k;
1299                 // <Quantitative ref="c2">
1300                 List<Element> elQuantitatives = elSummaryData.getChildren("Quantitative", sddNamespace);
1301                 k = 0;
1302                 //for each Quantitative
1303                 for (Element elQuantitative : elQuantitatives){
1304                         if ((++k % modCount) == 0){ logger.warn("Quantitative handled: " + (k-1));}
1305                         ref = elQuantitative.getAttributeValue("ref");
1306                         Feature feature = features.get(ref);
1307                         QuantitativeData quantitativeData = QuantitativeData.NewInstance();
1308                         quantitativeData.setFeature(feature);
1309
1310                         MeasurementUnit unit = units.get(ref);
1311                         String prefix = defaultUnitPrefixes.get(ref);
1312                         if (unit != null) {
1313                                 String u = unit.getLabel();
1314                                 if (prefix != null) {
1315                                         u = prefix + u;
1316                                 }
1317                                 unit.setLabel(u);
1318                                 quantitativeData.setUnit(unit);
1319                         }
1320
1321                         // <Measure type="Min" value="2.3"/>
1322                         List<Element> elMeasures = elQuantitative.getChildren("Measure", sddNamespace);
1323                         int l = 0;
1324
1325                         //for each State
1326                         for (Element elMeasure : elMeasures){
1327                                 if ((++l % modCount) == 0){ logger.info("States handled: " + (l-1));}
1328                                 String type = elMeasure.getAttributeValue("type");
1329                                 String value = elMeasure.getAttributeValue("value");
1330                                 if (value.contains(",")) {
1331                                         value = value.replace(',', '.');
1332                                 }
1333                                 Float v = Float.parseFloat(value);
1334                                 //Float v = new Float(0);
1335                                 StatisticalMeasure t = null;
1336                                 if (type.equals("Min")) {
1337                                         t = StatisticalMeasure.MIN();
1338                                 } else if (type.equals("Mean")) {
1339                                         t = StatisticalMeasure.AVERAGE();
1340                                 } else if (type.equals("Max")) {
1341                                         t = StatisticalMeasure.MAX();
1342                                 } else if (type.equals("SD")) {
1343                                         t = StatisticalMeasure.STANDARD_DEVIATION();
1344                                 } else if (type.equals("N")) {
1345                                         t = StatisticalMeasure.SAMPLE_SIZE();
1346                                 } else if (type.equals("UMethLower")) {
1347                                         t = StatisticalMeasure.TYPICAL_LOWER_BOUNDARY();
1348                                 } else if (type.equals("UMethUpper")) {
1349                                         t = StatisticalMeasure.TYPICAL_UPPER_BOUNDARY();
1350                                 } else if (type.equals("Var")) {
1351                                         t = StatisticalMeasure.VARIANCE();
1352                                 } else {
1353                                         t = StatisticalMeasure.NewInstance(type,type,type);
1354                                         statisticalMeasures.add(t);
1355                                 }
1356
1357                                 StatisticalMeasurementValue statisticalValue = StatisticalMeasurementValue.NewInstance();
1358                                 statisticalValue.setValue(v);
1359                                 statisticalValue.setType(t);
1360                                 quantitativeData.addStatisticalValue(statisticalValue);
1361                                 featureData.add(statisticalValue);
1362                         }
1363                         taxonDescription.addElement(quantitativeData);
1364                 }
1365         }
1366
1367         /**
1368          * @param sddNamespace
1369          * @param taxonDescription
1370          * @param elSummaryData
1371          */
1372         private void handleSummaryCategoricalData(Namespace sddNamespace,
1373                         TaxonDescription taxonDescription, Element elSummaryData) {
1374                 String ref;
1375                 // <Categorical ref="c4">
1376                 List<Element> elCategoricals = elSummaryData.getChildren("Categorical", sddNamespace);
1377                 int k = 0;
1378                 //for each Categorical
1379                 for (Element elCategorical : elCategoricals){
1380                         if ((++k % modCount) == 0){ logger.warn("Categorical handled: " + (k-1));}
1381                         ref = elCategorical.getAttributeValue("ref");
1382                         Feature feature = features.get(ref);
1383                         CategoricalData categoricalData = CategoricalData.NewInstance();
1384                         categoricalData.setFeature(feature);
1385
1386                         // <State ref="s3"/>
1387                         List<Element> elStates = elCategorical.getChildren("State", sddNamespace);
1388                         int l = 0;
1389
1390                         //for each State
1391                         for (Element elState : elStates){
1392                                 if ((++l % modCount) == 0){ logger.info("States handled: " + (l-1));}
1393                                 ref = elState.getAttributeValue("ref");
1394                                 State state = states.get(ref);
1395                                 if (state != null) {
1396                                         StateData stateData = StateData.NewInstance();
1397                                         stateData.setState(state);
1398                                         List<Element> elModifiers = elState.getChildren("Modifier", sddNamespace);
1399                                         for (Element elModifier : elModifiers){
1400                                                 ref = elModifier.getAttributeValue("ref");
1401                                                 Modifier modifier = modifiers.get(ref);
1402                                                 if (modifier != null) {
1403                                                         stateData.addModifier(modifier);
1404                                                 }
1405                                         }
1406                                         categoricalData.addState(stateData);
1407                                 }
1408                                 taxonDescription.addElement(categoricalData);
1409                         }
1410                 }
1411         }
1412
1413         // imports the persons associated with the dataset creation, modification, related publications
1414         protected void importAgents(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig, boolean success){
1415                 // <Agents>
1416                 logger.info("start Agents ...");
1417                 Element elAgents = elDataset.getChild("Agents",sddNamespace);
1418                 if (elAgents != null) {
1419                         // <Agent id="a1">
1420                         List <Element> listAgents = elAgents.getChildren("Agent", sddNamespace);
1421                         int j = 0;
1422                         //for each Agent
1423                         for (Element elAgent : listAgents){
1424
1425                                 try {
1426
1427                                         String idA = elAgent.getAttributeValue("id");
1428
1429                                         //  <Representation>
1430                                         //   <Label>Kevin Thiele</Label>
1431                                         //   <Detail role="Description">Ali Baba is also known as r.a.m.</Detail>
1432                                         //  </Representation>
1433                                         Person person = Person.NewInstance();
1434                                         importRepresentation(elAgent, sddNamespace, person, idA, sddConfig);
1435                                         person.addSource(IdentifiableSource.NewInstance(idA, "Agent"));
1436
1437                                         /*XIM <Links>
1438                                         Element elLinks = elAgent.getChild("Links",sddNamespace);
1439
1440                                         if (elLinks != null) {
1441
1442                                                 //  <Link rel="Alternate" href="http://www.diversitycampus.net/people/hagedorn"/>
1443                                                 List<Element> listLinks = elLinks.getChildren("Link", sddNamespace);
1444                                                 int k = 0;
1445                                                 //for each Link
1446                                                 for (Element elLink : listLinks){
1447
1448                                                         try {
1449
1450                                                                 String rel = elLink.getAttributeValue("rel");
1451                                                                 String href = elLink.getAttributeValue("href");
1452
1453                                                                 Media link = Media.NewInstance();
1454                                                                 MediaRepresentation mr = MediaRepresentation.NewInstance();
1455                                                                 mr.addRepresentationPart(MediaRepresentationPart.NewInstance(href, null));
1456                                                                 link.addRepresentation(mr);
1457                                                                 person.addMedia(link);
1458
1459                                                         } catch (Exception e) {
1460                                                                 //FIXME
1461                                                                 logger.warn("Import of Link " + k + " failed.");
1462                                                                 success = false;
1463                                                         }
1464
1465                                                         if ((++k % modCount) == 0){ logger.info("Links handled: " + k);}
1466
1467                                                 }
1468                                         }
1469                                         */
1470                                         if (authors.containsKey(idA)) {
1471                                                 authors.put(idA,person);
1472                                         }
1473
1474                                         if (editors.containsKey(idA)) {
1475                                                 editors.put(idA, person);
1476                                         }
1477
1478                                 } catch (Exception e) {
1479                                         //FIXME
1480                                         logger.warn("Import of Agent " + j + " failed.");
1481                                         success = false;
1482                                 }
1483
1484                                 if ((++j % modCount) == 0){ logger.info("Agents handled: " + j);}
1485
1486                         }
1487                 }
1488         }
1489
1490         // imports publications related with the data set
1491         protected void importPublications(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig, boolean success){
1492                 /* <Publications>
1493                           <Publication id="p112">
1494                             <Representation>
1495                               <Label>Gee, X. & Haa, Y. (2003). How to be happy in five minutes. Instant Gratifications, Palm Beach.</Label>
1496                             </Representation>
1497                             <Links>
1498                             <Link rel="BasedOn" href="doi:10.1992/32311"/>
1499                             <Link rel="Alternate" href="http://some.service.net/providing/bibliographic.data"/>
1500                             </Links>
1501                         </Publications>
1502 */
1503                 logger.info("start Publications ...");
1504                 Element elPublications = elDataset.getChild("Publications",sddNamespace);
1505
1506                 if (elPublications != null) {
1507                         List<Element> listPublications = elPublications.getChildren("Publication", sddNamespace);
1508                         int j = 0;
1509                         for (Element elPublication : listPublications){
1510
1511                                 try {
1512
1513                                         String idP = elPublication.getAttributeValue("id");
1514                                         ReferenceBase publication = ReferenceFactory.newArticle();
1515                                         importRepresentation(elPublication, sddNamespace, publication, idP, sddConfig);
1516
1517                                         publications.put(idP,publication);
1518
1519                                 } catch (Exception e) {
1520                                         logger.warn("Import of Publication " + j + " failed.");
1521                                         success = false;
1522                                 }
1523
1524                                 if ((++j % modCount) == 0){ logger.info("Publications handled: " + j);}
1525
1526                         }
1527                 }
1528         }
1529
1530         // imports media objects such as images //FIXME check mediaobj
1531         protected void importMediaObjects(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig, boolean success){
1532                 // <MediaObjects>
1533                 logger.info("start MediaObjects ...");
1534                 Element elMediaObjects = elDataset.getChild("MediaObjects",sddNamespace);
1535
1536                 if (elMediaObjects != null) {
1537                         // <MediaObject id="m1">
1538                         List<Element> listMediaObjects = elMediaObjects.getChildren("MediaObject", sddNamespace);
1539                         int j = 0;
1540                         for (Element elMO : listMediaObjects){
1541
1542                                 String id = "";
1543
1544                                 try {
1545                                         String idMO = elMO.getAttributeValue("id");
1546                                         id = idMO;
1547
1548                                         //  <Representation>
1549                                         //   <Label>Image description, e.g. to be used for alt-attribute in html.</Label>
1550                                         //  </Representation>
1551                                         Media media = Media.NewInstance();
1552                                         importRepresentation(elMO, sddNamespace, media, idMO, sddConfig);
1553
1554                                         // <Type>Image</Type>
1555                                         // <Source href="http://test.edu/test.jpg"/>
1556                                         String type = (String)ImportHelper.getXmlInputValue(elMO,"Type",sddNamespace);
1557
1558                                         if ((type != null) && (type.equals("Image"))) {
1559                                                 Element elSource = elMO.getChild("Source",sddNamespace);
1560                                                 String href = elSource.getAttributeValue("href");
1561
1562                                                 ImageMetaData imageMetaData = ImageMetaData.newInstance();
1563                                                 ImageFile image = null;
1564                                                 if (href.substring(0,7).equals("http://")) {
1565                                                         try{
1566                                                                 URL url = new URL(href);
1567
1568                                                                 imageMetaData.readMetaData(url.toURI(), 0);
1569                                                                 image = ImageFile.NewInstance(url.toString(), null, imageMetaData);
1570                                                         } catch (MalformedURLException e) {
1571                                                                 logger.error("Malformed URL", e);
1572                                                         }
1573                                                 } else {
1574                                                         String sns = sddConfig.getSourceNameString();
1575                                                         File f = new File(sns);
1576                                                         File parent = f.getParentFile();
1577                                                         String fi = href;
1578                                                         //String fi = parent.toString() + File.separator + href; //TODO erase file:/
1579                                                         File file = new File(fi);
1580                                                         imageMetaData.readMetaData(file.toURI(), 0);
1581                                                         image = ImageFile.NewInstance(file.toString(), null, imageMetaData);
1582                                                 }
1583                                                 MediaRepresentation representation = MediaRepresentation.NewInstance(imageMetaData.getMimeType(), null);
1584                                                 representation.addRepresentationPart(image);
1585
1586                                                 media.addRepresentation(representation);
1587
1588                                                 ArrayList<CdmBase> lcb = (ArrayList<CdmBase>) mediaObject_ListCdmBase.get(idMO);
1589                                                 if (lcb != null) {
1590                                                         for (int k = 0; k < lcb.size(); k++) {
1591                                                                 if (lcb.get(k) instanceof DefinedTermBase) {
1592                                                                         DefinedTermBase dtb = (DefinedTermBase) lcb.get(k);
1593                                                                         // if (lcb.get(0) instanceof DefinedTermBase) {
1594                                                                         // DefinedTermBase dtb = (DefinedTermBase) lcb.get(0);
1595                                                                         //                                                                      if (dtb!=null) {
1596                                                                         //                                                                              if (k == 0) {
1597                                                                         dtb.addMedia(media);
1598                                                                         //System.out.println(dtb.getLabel());
1599                                                                         //                                                                              } else {
1600                                                                         //                                                                                      Media me = (Media) media.clone();
1601                                                                         //                                                                                      dtb.addMedia(me);
1602                                                                         //                                                                              }
1603                                                                         //                                                                      }
1604                                                                 } else if (lcb.get(k) instanceof ReferenceBase) {
1605                                                                         ReferenceBase rb = (ReferenceBase) lcb.get(k);
1606                                                                         //} else if (lcb.get(0) instanceof ReferenceBase) {
1607                                                                         //ReferenceBase rb = (ReferenceBase) lcb.get(0);
1608                                                                         // rb.setTitleCache(label);
1609                                                                         //                                                                      if (rb!=null) {
1610                                                                         //                                                                              if (k == 0) {
1611                                                                         rb.addMedia(media);
1612                                                                         //System.out.println(rb.getTitle());
1613                                                                         //                                                                              } else {
1614                                                                         //                                                                                      Media me = (Media) media.clone();
1615                                                                         //                                                                                      rb.addMedia(me);
1616                                                                         //                                                                              }
1617                                                                         //                                                                      }
1618                                                                 }/* else if (lcb.get(k) instanceof TaxonNameBase){
1619                                                                         TaxonNameBase tb = (TaxonNameBase) lcb.get(k);
1620                                                                         tb.addMedia(media);
1621                                                         }*/
1622                                                         }
1623                                                 }
1624                                         }
1625
1626                                 } catch (Exception e) {
1627                                         //FIXME
1628                                         logger.warn("Could not attach MediaObject " + j + "(SDD: " + id + ") to several objects.");
1629                                         success = false;
1630                                 }
1631
1632                                 if ((++j % modCount) == 0){ logger.info("MediaObjects handled: " + j);
1633
1634                                 }
1635                         }
1636                 }
1637         }
1638
1639         // imports the <DescriptiveConcepts> block ; DescriptiveConcepts are used as nodes in CharacterTrees and Characters as leaves
1640         // but since Modifiers can be linked to DescriptiveConcepts they are stored as features with a particular Marker
1641         protected void importDescriptiveConcepts(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig){
1642                 /* <DescriptiveConcepts>
1643                       <DescriptiveConcept id="dc0">
1644                                 <Representation>
1645                                   <Label>Fixed set of modifiers supported in Lucid3</Label>
1646                                 </Representation>
1647                                 <Modifiers>
1648                                   <Modifier id="mod1">
1649                                     <Representation>
1650                                       <Label>rarely</Label>
1651                                     </Representation>
1652                                     <ModifierClass>Frequency</ModifierClass>
1653                                     <ProportionRange lowerestimate="0.0" upperestimate="0.25"/>
1654                                   </Modifier>
1655                           </Modifiers>
1656                         </DescriptiveConcept>
1657                  </DescriptiveConcepts>
1658                  */
1659                 logger.info("start DescriptiveConcepts ...");
1660                 Element elDescriptiveConcepts = elDataset.getChild("DescriptiveConcepts",sddNamespace);
1661                 if (elDescriptiveConcepts != null) {
1662                         List<Element> listDescriptiveConcepts = elDescriptiveConcepts.getChildren("DescriptiveConcept", sddNamespace);
1663                         int j = 0;
1664
1665                         for (Element elDescriptiveConcept : listDescriptiveConcepts){
1666                                 try {
1667                                 String id = elDescriptiveConcept.getAttributeValue("id");
1668                                         Feature feature = Feature.NewInstance();
1669                                         feature.setKindOf(Feature.DESCRIPTION());
1670                                         if (!id.equals("")) {
1671                                         //       <Representation>
1672                                         //       <Label>Body</Label>
1673                                         importRepresentation(elDescriptiveConcept, sddNamespace, feature, id, sddConfig);
1674                                                 features.put(id, feature);
1675                                                 getTermService().save(feature);//XIM
1676                                                 descriptiveConcepts.add(feature);
1677                                                 // imports the modifiers
1678                                                 Element elModifiers = elDescriptiveConcept.getChild("Modifiers", sddNamespace);
1679                                         if (elModifiers !=null){
1680                                                 List<Element> listModifiers = elModifiers.getChildren("Modifier", sddNamespace);
1681                                                         TermVocabulary<Modifier> termVocabularyState = new TermVocabulary<Modifier>();
1682                                                 for (Element elModifier : listModifiers) {
1683                                                                 Modifier modif = Modifier.NewInstance();
1684                                                                 String idmod = elModifier.getAttributeValue("id");
1685                                                                 importRepresentation(elModifier, sddNamespace, modif, idmod, sddConfig);
1686                                                                 termVocabularyState.addTerm(modif);
1687                                                                 //termVocabularyStates.add(termVocabularyState);
1688                                                                 getVocabularyService().save(termVocabularyState);//XIM
1689                                                                 modifiers.put(idmod, modif);
1690                                                 }
1691                                                         feature.addRecommendedModifierEnumeration(termVocabularyState);
1692                                 }
1693
1694                                         }
1695                                 }
1696                                 catch (Exception e) {
1697                                         logger.warn("Import of DescriptiveConcept " + j + " failed.");
1698                                 }
1699                                 if ((++j % modCount) == 0){ logger.info("DescriptiveConcepts handled: " + j);}
1700
1701                         }
1702                 }
1703         }
1704
1705         // imports the <CharacterTrees> block
1706         protected void importCharacterTrees(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig, boolean success){
1707                 // <CharacterTrees>
1708                 logger.info("start CharacterTrees ...");
1709                 Element elCharacterTrees = elDataset.getChild("CharacterTrees",sddNamespace);
1710
1711                 if (elCharacterTrees != null) {
1712                         List<Element> listCharacterTrees = elCharacterTrees.getChildren("CharacterTree", sddNamespace);
1713                         int j = 0;
1714                         for (Element elCharacterTree : listCharacterTrees){
1715                                 try {
1716                                         Element elRepresentation = elCharacterTree.getChild("Representation",sddNamespace);
1717                                         String label = (String)ImportHelper.getXmlInputValue(elRepresentation,"Label",sddNamespace);
1718                                         //Element elDesignedFor = elCharacterTree.getChild("DesignedFor",sddNamespace);//TODO ?
1719
1720                                         FeatureTree feattree =  FeatureTree.NewInstance();
1721                                         importRepresentation(elCharacterTree, sddNamespace, feattree, "", sddConfig);
1722                                         FeatureNode root = feattree.getRoot();
1723                                         List<Element> listelNodes = elCharacterTree.getChildren("Nodes", sddNamespace);
1724
1725                                         //Nodes of CharacterTrees in SDD always refer to DescriptiveConcepts
1726                                         for (Element elNodes : listelNodes) {
1727                                                 handleCharacterNodes(sddNamespace, root, elNodes);
1728                                         }
1729                                         featureTrees.add(feattree);
1730                                 }
1731
1732                                 catch (Exception e) {
1733                                         logger.warn("Import of Character tree " + j + " failed.");
1734                                         success = false;
1735                                 }
1736                                 if ((++j % modCount) == 0){ logger.info("CharacterTrees handled: " + j);}
1737
1738                         }
1739
1740                 }
1741         }
1742
1743         /**
1744          * @param sddNamespace
1745          * @param root
1746          * @param elNodes
1747          */
1748         private void handleCharacterNodes(Namespace sddNamespace, FeatureNode root, Element elNodes) {
1749                 List<Element> listNodes = elNodes.getChildren("Node", sddNamespace);
1750                 if (listNodes != null) {
1751                         for (Element elNode : listNodes){
1752                                 String idN = elNode.getAttributeValue("id");
1753                                 FeatureNode fn = null;
1754                                 Feature dc = null;
1755                                 if (idN!=null) {
1756                                         // DescriptiveConcepts are used as nodes in CharacterTrees
1757                                         Element elDescriptiveConcept = elNode.getChild("DescriptiveConcept", sddNamespace);
1758                                         if (elDescriptiveConcept != null){
1759                                                 String refDC = elDescriptiveConcept.getAttributeValue("ref");
1760                                                 dc = features.get(refDC);
1761                                                 fn = FeatureNode.NewInstance(dc);
1762                                         }
1763                                         if (fn==null){
1764                                                 fn = FeatureNode.NewInstance();
1765                                         }
1766                                         Element elParent = elNode.getChild("Parent", sddNamespace);
1767                                         // in SDD links between Nodes are referenced by the <Parent> tag
1768                                         if (elParent!=null){
1769                                                 String refP = elParent.getAttributeValue("ref");
1770                                                 if (refP!=null) {
1771                                                         FeatureNode parent = featureNodes.get(refP);
1772                                                         if (parent==null){
1773                                                                 root.addChild(fn); // if no parent found or the reference is broken, add the node to the root of the tree
1774                                                         }
1775                                                         else {
1776                                                                 parent.addChild(fn);
1777                                                         }
1778                                                 }
1779                                         }
1780                                         else {
1781                                                 root.addChild(fn); // if no parent found or the reference is broken, add the node to the root of the tree
1782                                         }
1783                                 }
1784                                 featureNodes.put(idN, fn);
1785                         }
1786                 }
1787
1788                 // Leaves of CharacterTrees in SDD are always CharNodes (referring to Characters)
1789                 List<Element> listCharNodes = elNodes.getChildren("CharNode", sddNamespace);
1790                 if (listCharNodes != null) {
1791                         for (Element elCharNode : listCharNodes){
1792                                 Element elParent = elCharNode.getChild("Parent", sddNamespace);
1793                                 Element elCharacter = elCharNode.getChild("Character", sddNamespace);
1794                                 FeatureNode fn = FeatureNode.NewInstance();
1795                                 if (elParent!=null){
1796                                         String refP = elParent.getAttributeValue("ref");
1797                                         if ((refP!=null)&&(!refP.equals(""))) {
1798                                         FeatureNode parent = featureNodes.get(refP);
1799                                                 if (parent==null){
1800                                                 parent = root; // if no parent found or the reference is broken, add the node to the root of the tree
1801                                                 }
1802                                                 parent.addChild(fn);
1803                                         }
1804                                 }
1805                                 String refC = elCharacter.getAttributeValue("ref");
1806                                 if ((refC!=null)&&(!refC.equals(""))){
1807                                         Feature character = features.get(refC);
1808                                         fn.setFeature(character);
1809                                         featureNodes.put(refC, fn);
1810                                 }
1811                         }
1812                 }
1813         }
1814
1815         // imports the <TaxonHierarchies> block
1816         protected void importTaxonHierarchies(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig, boolean success){
1817
1818                 logger.info("start TaxonHierarchies ...");
1819                 Element elTaxonHierarchies = elDataset.getChild("TaxonHierarchies",sddNamespace);
1820
1821                 if (elTaxonHierarchies != null) {
1822                         List<Element> listTaxonHierarchies = elTaxonHierarchies.getChildren("TaxonHierarchy", sddNamespace);
1823                         int j = 0;
1824                         for (Element elTaxonHierarchy : listTaxonHierarchies){
1825                                 try {
1826                                         Element elRepresentation = elTaxonHierarchy.getChild("Representation",sddNamespace);
1827                                         String label = (String)ImportHelper.getXmlInputValue(elRepresentation,"Label",sddNamespace);
1828                                                 TaxonomicTree taxonomicTree =  TaxonomicTree.NewInstance(label);
1829                                                 importRepresentation(elTaxonHierarchy, sddNamespace, taxonomicTree, "", sddConfig);
1830
1831                                                 Set<TaxonNode> root = taxonomicTree.getChildNodes();
1832                                                 Element elNodes = elTaxonHierarchy.getChild("Nodes", sddNamespace); // There can be only one <Nodes> block for TaxonHierarchies
1833                                                 List<Element> listNodes = elNodes.getChildren("Node", sddNamespace);
1834
1835                                                 for (Element elNode : listNodes){
1836                                                         String idN = elNode.getAttributeValue("id");
1837                                                         TaxonNameBase tnb = null;
1838                                                         if (!idN.equals("")) {
1839                                                                 Element elTaxonName = elNode.getChild("TaxonName", sddNamespace);
1840                                                                 String refTN = elTaxonName.getAttributeValue("ref");
1841                                                                 tnb = taxonNameBases.get(refTN);
1842                                                                 Taxon taxon = (Taxon) tnb.getTaxa().iterator().next() ;
1843                                                                 Element elParent = elNode.getChild("Parent", sddNamespace);
1844                                                                 if (elParent!=null){
1845                                                                         String refP = elParent.getAttributeValue("ref");
1846                                                                         if (!refP.equals("")) {
1847                                                                                 TaxonNode parent = taxonNodes.get(refP);
1848                                                                                 TaxonNode child = parent.addChildTaxon(taxon, sec, "", Synonym.NewInstance(tnb, sec));
1849                                                                                 taxonNodes.put(idN,child);
1850                                                                         }
1851                                                                 }
1852                                                                 else {
1853                                                                         TaxonNode tn = taxonomicTree.addChildTaxon(taxon, sec, "", Synonym.NewInstance(tnb, sec)); // if no parent found or the reference is broken, add the node to the root of the tree
1854                                                                         taxonNodes.put(idN,tn);
1855                                                                 }
1856                                                         }
1857                                                 }
1858
1859                                                 taxonomicTrees.add(taxonomicTree);
1860                                         }
1861
1862                                 catch (Exception e) {
1863                                         //FIXME
1864                                         logger.warn("Import of Taxon Hierarchy " + j + " failed.");
1865                                         success = false;
1866                                 }
1867
1868                                 if ((++j % modCount) == 0){ logger.info("TaxonHierarchies handled: " + j);}
1869
1870                         }
1871
1872                 }
1873         }
1874
1875
1876         // imports the <GeographicAreas> block
1877         protected void importGeographicAreas(Element elDataset, Namespace sddNamespace, SDDImportConfigurator sddConfig) {
1878                 Element elGeographicAreas = elDataset.getChild("GeographicAreas",sddNamespace);
1879                 if (elGeographicAreas != null) {
1880                         List<Element> listGeographicAreas = elGeographicAreas.getChildren("GeographicArea", sddNamespace);
1881                         int j = 0;
1882
1883                         for (Element elGeographicArea : listGeographicAreas){
1884
1885                                 String id = elGeographicArea.getAttributeValue("id");
1886                                 NamedArea na = new NamedArea();
1887                                 importRepresentation(elGeographicArea, sddNamespace, na, id, sddConfig);
1888                                 namedAreas.put(id,na);
1889                                                                 }
1890                         if ((++j % modCount) == 0){ logger.info("GeographicAreas handled: " + j);}
1891                                                         }
1892                                                         }
1893 }