cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupDocumentImportNoComponent.java

   1 /**
   2  * Copyright (C) 2009 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.markup;
  11
  12 import java.net.MalformedURLException;
  13 import java.net.URL;
  14 import java.util.ArrayList;
  15 import java.util.Arrays;
  16 import java.util.HashMap;
  17 import java.util.HashSet;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Set;
  21 import java.util.UUID;
  22 import java.util.regex.Matcher;
  23 import java.util.regex.Pattern;
  24
  25 import javax.xml.stream.Location;
  26 import javax.xml.stream.XMLEventReader;
  27 import javax.xml.stream.XMLStreamException;
  28 import javax.xml.stream.events.Attribute;
  29 import javax.xml.stream.events.StartElement;
  30 import javax.xml.stream.events.XMLEvent;
  31
  32 import org.apache.commons.lang.StringUtils;
  33 import org.apache.log4j.Logger;
  34
  35 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  36 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
  37 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacadeCacheStrategy;
  38 import eu.etaxonomy.cdm.common.CdmUtils;
  39 import eu.etaxonomy.cdm.ext.geo.GeoServiceArea;
  40 import eu.etaxonomy.cdm.ext.geo.IEditGeoService;
  41 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  42 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  43 import eu.etaxonomy.cdm.io.markup.UnmatchedLeads.UnmatchedLeadsKey;
  44 import eu.etaxonomy.cdm.model.agent.AgentBase;
  45 import eu.etaxonomy.cdm.model.agent.INomenclaturalAuthor;
  46 import eu.etaxonomy.cdm.model.agent.Team;
  47 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  48 import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
  49 import eu.etaxonomy.cdm.model.common.Annotation;
  50 import eu.etaxonomy.cdm.model.common.AnnotationType;
  51 import eu.etaxonomy.cdm.model.common.CdmBase;
  52 import eu.etaxonomy.cdm.model.common.Extension;
  53 import eu.etaxonomy.cdm.model.common.ExtensionType;
  54 import eu.etaxonomy.cdm.model.common.Language;
  55 import eu.etaxonomy.cdm.model.common.TermVocabulary;
  56 import eu.etaxonomy.cdm.model.common.TimePeriod;
  57 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  58 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  59 import eu.etaxonomy.cdm.model.description.Distribution;
  60 import eu.etaxonomy.cdm.model.description.Feature;
  61 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  62 import eu.etaxonomy.cdm.model.description.KeyStatement;
  63 import eu.etaxonomy.cdm.model.description.PolytomousKey;
  64 import eu.etaxonomy.cdm.model.description.PolytomousKeyNode;
  65 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTermBase;
  66 import eu.etaxonomy.cdm.model.description.PresenceTerm;
  67 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  68 import eu.etaxonomy.cdm.model.description.TextData;
  69 import eu.etaxonomy.cdm.model.location.NamedArea;
  70 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  71 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  72 import eu.etaxonomy.cdm.model.media.IdentifiableMediaEntity;
  73 import eu.etaxonomy.cdm.model.media.Media;
  74 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  75 import eu.etaxonomy.cdm.model.name.HomotypicalGroup;
  76 import eu.etaxonomy.cdm.model.name.NameTypeDesignationStatus;
  77 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  78 import eu.etaxonomy.cdm.model.name.NomenclaturalStatus;
  79 import eu.etaxonomy.cdm.model.name.NomenclaturalStatusType;
  80 import eu.etaxonomy.cdm.model.name.NonViralName;
  81 import eu.etaxonomy.cdm.model.name.Rank;
  82 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  83 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  84 import eu.etaxonomy.cdm.model.occurrence.Collection;
  85 import eu.etaxonomy.cdm.model.occurrence.DerivedUnitBase;
  86 import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
  87 import eu.etaxonomy.cdm.model.occurrence.Specimen;
  88 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  89 import eu.etaxonomy.cdm.model.reference.IArticle;
  90 import eu.etaxonomy.cdm.model.reference.IJournal;
  91 import eu.etaxonomy.cdm.model.reference.Reference;
  92 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  93 import eu.etaxonomy.cdm.model.reference.ReferenceType;
  94 import eu.etaxonomy.cdm.model.taxon.Classification;
  95 import eu.etaxonomy.cdm.model.taxon.SynonymRelationshipType;
  96 import eu.etaxonomy.cdm.model.taxon.Taxon;
  97 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  98 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
  99 import eu.etaxonomy.cdm.strategy.parser.NameTypeParser;
 100 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
 101 import eu.etaxonomy.cdm.strategy.parser.SpecimenTypeParser;
 102 import eu.etaxonomy.cdm.strategy.parser.SpecimenTypeParser.TypeInfo;
 103
 104 /**
 105  * @author a.mueller
 106  *
 107  */
 108 public class MarkupDocumentImportNoComponent extends MarkupImportBase {
 109         private static final Logger logger = Logger.getLogger(MarkupDocumentImportNoComponent.class);
 110
 111         private static final boolean CREATE_NEW = true;
 112         private static final boolean IS_IMAGE_GALLERY = true;
 113         private static final boolean NO_IMAGE_GALLERY = false;
 114
 115
 116         private static final String ACCEPTED = "accepted";
 117         private static final String ACCEPTED_NAME = "acceptedName";
 118         private static final String ADDENDA = "addenda";
 119         private static final String ALTERNATEPUBTITLE = "alternatepubtitle";
 120         private static final String ALTERNATIVE_COLLECTION_TYPE_STATUS = "alternativeCollectionTypeStatus";
 121         private static final String ALTERNATIVE_COLLECTOR = "alternativeCollector";
 122         private static final String ALTERNATIVE_FIELD_NUM = "alternativeFieldNum";
 123         private static final String ALTITUDE = "altitude";
 124         private static final String ANNOTATION = "annotation";
 125         private static final String AUTHOR = "author";
 126         private static final String BIBLIOGRAPHY = "bibliography";
 127         private static final String BIOGRAPHIES = "biographies";
 128         private static final String BOLD = "bold";
 129         private static final String BR = "br";
 130         private static final String CHAR = "char";
 131         private static final String CITATION = "citation";
 132         private static final String COLLECTION_AND_TYPE = "collectionAndType";
 133         private static final String COLLECTION_TYPE_STATUS = "collectionTypeStatus";
 134         private static final String COLLECTOR = "collector";
 135         private static final String COLLECTION = "collection";
 136         private static final String COORDINATES = "coordinates";
 137         private static final String COUPLET = "couplet";
 138         private static final String DATES = "dates";
 139         private static final String DEDICATION = "dedication";
 140         private static final String DEFAULT_MEDIA_URL = "defaultMediaUrl";
 141         private static final String DESTROYED = "destroyed";
 142         private static final String DETAILS = "details";
 143         private static final String DISTRIBUTION_LIST = "distributionList";
 144         private static final String DISTRIBUTION_LOCALITY = "distributionLocality";
 145         private static final String EDITION = "edition";
 146         private static final String EDITORS = "editors";
 147         private static final String FEATURE = "feature";
 148         private static final String FIGURE = "figure";
 149         private static final String FIGURE_LEGEND = "figureLegend";
 150         private static final String FIGURE_PART = "figurePart";
 151         private static final String FIGURE_REF = "figureRef";
 152         private static final String FIGURE_TITLE = "figureTitle";
 153         private static final String FOOTNOTE = "footnote";
 154         private static final String FOOTNOTE_REF = "footnoteRef";
 155         private static final String FOOTNOTE_STRING = "footnoteString";
 156         private static final String FIELD_NUM = "fieldNum";
 157         private static final String FREQUENCY = "frequency";
 158         private static final String FULL_NAME = "fullName";
 159         private static final String FULL_TYPE = "fullType";
 160         private static final String GATHERING = "gathering";
 161         private static final String HEADING = "heading";
 162         private static final String HABITAT = "habitat";
 163         private static final String HABITAT_LIST = "habitatList";
 164         private static final String HOMONYM = "homonym";
 165         private static final String HOMOTYPES = "homotypes";
 166         private static final String ID = "id";
 167         private static final String INFRANK = "infrank";
 168         private static final String INFRAUT = "infraut";
 169         private static final String INFRPARAUT = "infrparaut";
 170         private static final String IS_SPOTCHARACTERS = "isSpotcharacters";
 171         private static final String ISSUE = "issue";
 172         private static final String ITALICS = "italics";
 173         private static final String KEY = "key";
 174         private static final String KEY_TITLE = "keyTitle";
 175         private static final String KEYNOTES = "keynotes";
 176         private static final String LIFE_CYCLE_PERIODS = "lifeCyclePeriods";
 177         private static final String LOCALITY = "locality";
 178         private static final String LOST = "lost";
 179         private static final String META_DATA = "metaData";
 180         private static final String NAME = "name";
 181         private static final String NAME_TYPE = "nameType";
 182         private static final String NOM = "nom";
 183         private static final String NOMENCLATURE = "nomenclature";
 184         private static final String NOT_FOUND = "notFound";
 185         private static final String NOT_SEEN = "notSeen";
 186         private static final String NOTES = "notes";
 187         private static final String NUM = "num";
 188         private static final String ORIGINAL_DETERMINATION = "originalDetermination";
 189         private static final String PAGES = "pages";
 190         private static final String PARAUT = "paraut";
 191         private static final String PUBFULLNAME = "pubfullname";
 192         private static final String PUBLICATION = "publication";
 193         private static final String PUBNAME = "pubname";
 194         private static final String PUBTITLE = "pubtitle";
 195         private static final String PUBTYPE = "pubtype";
 196         private static final String QUESTION = "question";
 197         private static final String QUOTE = "quote";
 198         private static final String RANK = "rank";
 199         private static final String REF = "ref";
 200         private static final String REF_NUM = "refNum";
 201         private static final String REF_PART = "refPart";
 202         private static final String REFERENCE = "reference";
 203         private static final String REFERENCES = "references";
 204         private static final String TAXON = "taxon";
 205         private static final String TAXONTITLE = "taxontitle";
 206         private static final String TAXONTYPE = "taxontype";
 207         private static final String TEXT = "text";
 208         private static final String TEXT_SECTION = "textSection";
 209         private static final String TO_COUPLET = "toCouplet";
 210         private static final String TO_KEY = "toKey";
 211         private static final String TO_TAXON = "toTaxon";
 212         private static final String TYPE = "type";
 213         private static final String TYPE_STATUS = "typeStatus";
 214         private static final String TREATMENT = "treatment";
 215         private static final String SERIALS_ABBREVIATIONS = "serialsAbbreviations";
 216         private static final String SPECIMEN_TYPE = "specimenType";
 217         private static final String STATUS = "status";
 218         private static final String STRING = "string";
 219         private static final String SUB_HEADING = "subHeading";
 220         private static final String SUB_COLLECTION = "subCollection";
 221         private static final String SYNONYM = "synonym";
 222         private static final String UNKNOWN = "unknown";
 223         private static final String URL = "url";
 224         private static final String USAGE = "usage";
 225         private static final String VOLUME = "volume";
 226         private static final String WRITER = "writer";
 227         private static final String YEAR = "year";
 228
 229         private NonViralNameParserImpl parser = new NonViralNameParserImpl();
 230
 231         // TODO make part of state, but state is renewed when invoking the import a
 232         // second time
 233         private UnmatchedLeads unmatchedLeads;
 234
 235
 236         private IEditGeoService editGeoService;
 237
 238         public MarkupDocumentImportNoComponent(MarkupDocumentImport docImport) {
 239                 super(docImport);
 240                 this.editGeoService = docImport.getEditGeoService();
 241         }
 242
 243         public void doInvoke(MarkupImportState state) throws XMLStreamException {
 244                 XMLEventReader reader = state.getReader();
 245
 246                 // publication
 247                 String elName = PUBLICATION;
 248                 boolean hasPublication = false;
 249
 250                 while (reader.hasNext()) {
 251                         XMLEvent nextEvent = reader.nextEvent();
 252                         if (isStartingElement(nextEvent, elName)) {
 253                                 handlePublication(state, reader, nextEvent, elName);
 254                                 hasPublication = true;
 255                         } else if (nextEvent.isEndDocument()) {
 256                                 if (!hasPublication) {
 257                                         String message = "No publication root element found";
 258                                         fireWarningEvent(message, nextEvent, 8);
 259                                 }
 260                                 // done
 261                         } else {
 262                                 fireSchemaConflictEventExpectedStartTag(elName, reader);
 263                         }
 264                 }
 265
 266
 267                 return;
 268
 269         }
 270
 271         private void handlePublication(MarkupImportState state, XMLEventReader reader, XMLEvent currentEvent,
 272                         String elName) throws XMLStreamException {
 273
 274                 // attributes
 275                 StartElement element = currentEvent.asStartElement();
 276                 Map<String, Attribute> attributes = getAttributes(element);
 277                 String lang = getAndRemoveAttributeValue(attributes, "lang");
 278                 if (lang != null){
 279                         Language language = getTermService().getLanguageByIso(lang);
 280                         state.setDefaultLanguage(language);
 281                 }
 282
 283                 handleUnexpectedAttributes(element.getLocation(), attributes, "noNamespaceSchemaLocation");
 284
 285                 while (reader.hasNext()) {
 286                         XMLEvent event = readNoWhitespace(reader);
 287                         // TODO cardinality of alternative
 288                         if (event.isEndElement()) {
 289                                 if (isEndingElement(event, elName)) {
 290                                         return;
 291                                 } else {
 292                                         if (isEndingElement(event, BIOGRAPHIES)) {
 293                                                 // NOT YET IMPLEMENTED
 294                                                 popUnimplemented(event.asEndElement());
 295                                         } else if (isEndingElement(event, REFERENCES)) {
 296                                                 // NOT YET IMPLEMENTED
 297                                                 popUnimplemented(event.asEndElement());
 298                                         } else if (isEndingElement(event, TEXT_SECTION)) {
 299                                                 // NOT YET IMPLEMENTED
 300                                                 popUnimplemented(event.asEndElement());
 301                                         } else if (isEndingElement(event, ADDENDA)) {
 302                                                 // NOT YET IMPLEMENTED
 303                                                 popUnimplemented(event.asEndElement());
 304                                         } else {
 305                                                 handleUnexpectedElement(event);
 306                                         }
 307                                 }
 308                         } else if (event.isStartElement()) {
 309                                 if (isStartingElement(event, META_DATA)) {
 310                                         handleMetaData(state, reader, event);
 311                                 } else if (isStartingElement(event, TREATMENT)) {
 312                                         handleTreatment(state, reader, event);
 313                                 } else if (isStartingElement(event, BIOGRAPHIES)) {
 314                                         handleNotYetImplementedElement(event);
 315                                 } else if (isStartingElement(event, REFERENCES)) {
 316                                         handleNotYetImplementedElement(event);
 317                                 } else if (isStartingElement(event, TEXT_SECTION)) {
 318                                         handleNotYetImplementedElement(event);
 319                                 } else if (isStartingElement(event, ADDENDA)) {
 320                                         handleNotYetImplementedElement(event);
 321                                 } else {
 322                                         handleUnexpectedStartElement(event);
 323                                 }
 324                         } else {
 325                                 handleUnexpectedElement(event);
 326                         }
 327                 }
 328                 throw new IllegalStateException("Publication has no ending element");
 329         }
 330
 331         private void handleMetaData(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
 332                 checkNoAttributes(parentEvent);
 333
 334                 while (reader.hasNext()) {
 335                         XMLEvent next = readNoWhitespace(reader);
 336                         if (isMyEndingElement(next, parentEvent)) {
 337                                 return;
 338                         } else if (isStartingElement(next, DEFAULT_MEDIA_URL)) {
 339                                 String baseUrl = getCData(state, reader, next);
 340                                 try {
 341                                         new URL(baseUrl);
 342                                         state.setBaseMediaUrl(baseUrl);
 343                                 } catch (MalformedURLException e) {
 344                                         String message = "defaultMediaUrl '%s' is not a valid URL";
 345                                         message = String.format(message, baseUrl);
 346                                         fireWarningEvent(message, next, 8);
 347                                 }
 348                         } else {
 349                                 handleUnexpectedElement(next);
 350                         }
 351                 }
 352                 throw new IllegalStateException("MetaData has no ending element");
 353
 354         }
 355
 356         private void handleTreatment(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
 357                 checkNoAttributes(parentEvent);
 358                 Taxon lastTaxon = null;
 359                 while (reader.hasNext()) {
 360                         XMLEvent next = readNoWhitespace(reader);
 361                         if (isStartingElement(next, TAXON)) {
 362                                 Taxon thisTaxon = handleTaxon(state, reader, next.asStartElement());
 363                                 doTaxonRelation(state, thisTaxon, lastTaxon, parentEvent.getLocation());
 364                                 lastTaxon = thisTaxon;
 365                                 // TODO for imports spanning multiple documents ?? Still needed?
 366                                 state.getConfig().setLastTaxonUuid(lastTaxon.getUuid());
 367                         } else if (isMyEndingElement(next, parentEvent)) {
 368                                 Set<PolytomousKeyNode> keyNodesToSave = state.getPolytomousKeyNodesToSave();
 369                                 //better save the key then the nodes
 370                                 Set<PolytomousKey> keySet = new HashSet<PolytomousKey>();
 371                                 for (PolytomousKeyNode node : keyNodesToSave){
 372                                         PolytomousKey key = node.getKey();
 373                                         keySet.add(key);
 374                                 }
 375                                 save(keySet, state);
 376                                 //unmatched key leads
 377                                 UnmatchedLeads unmatched = state.getUnmatchedLeads();
 378                                 if (unmatched.size() > 0){
 379                                         String message = "The following key leads are unmatched: %s";
 380                                         message = String.format(message, state.getUnmatchedLeads().toString());
 381                                         fireWarningEvent(message, next, 6);
 382                                 }
 383 //                              save(keyNodesToSave, state);
 384
 385                                 return;
 386                         } else {
 387                                 handleUnexpectedElement(next);
 388                         }
 389                 }
 390                 return;
 391         }
 392
 393         /**
 394          * @param taxon
 395          * @param lastTaxon
 396          */
 397         private void doTaxonRelation(MarkupImportState state, Taxon taxon,
 398                         Taxon lastTaxon, Location dataLocation) {
 399
 400                 Classification tree = makeTree(state, dataLocation);
 401                 if (lastTaxon == null) {
 402                         tree.addChildTaxon(taxon, null, null, null);
 403                         return;
 404                 }
 405                 Rank thisRank = taxon.getName().getRank();
 406                 Rank lastRank = lastTaxon.getName().getRank();
 407                 if (lastTaxon.getTaxonNodes().size() > 0) {
 408                         TaxonNode lastNode = lastTaxon.getTaxonNodes().iterator().next();
 409                         if (thisRank == null){
 410                                 String message = "rank is undefined for taxon '%s'. Can't create classification without rank.";
 411                                 message = String.format(message, taxon.getName().getTitleCache());
 412                                 fireWarningEvent(message, makeLocationStr(dataLocation), 6);
 413                         }else if (thisRank.isLower(lastRank)) {
 414                                 lastNode.addChildTaxon(taxon, null, null, null);
 415                                 fillMissingEpithetsForTaxa(lastTaxon, taxon);
 416                         } else if (thisRank.equals(lastRank)) {
 417                                 TaxonNode parent = lastNode.getParent();
 418                                 if (parent != null) {
 419                                         parent.addChildTaxon(taxon, null, null, null);
 420                                         fillMissingEpithetsForTaxa(parent.getTaxon(), taxon);
 421                                 } else {
 422                                         tree.addChildTaxon(taxon, null, null, null);
 423                                 }
 424                         } else if (thisRank.isHigher(lastRank)) {
 425                                 doTaxonRelation(state, taxon, lastNode.getParent().getTaxon(),  dataLocation);
 426                                 // TaxonNode parentNode = handleTaxonRelation(state, taxon,
 427                                 // lastNode.getParent().getTaxon());
 428                                 // parentNode.addChildTaxon(taxon, null, null, null);
 429                         }
 430                 } else {
 431
 432                         String message = "Last taxon has no node";
 433                         fireWarningEvent(message, makeLocationStr(dataLocation), 6);
 434                 }
 435         }
 436
 437
 438
 439         /**
 440          * @param state
 441          * @param dataLocation
 442          * @return
 443          */
 444         private Classification makeTree(MarkupImportState state, Location dataLocation) {
 445                 Classification result = state.getTree(null);
 446                 if (result == null) {
 447                         UUID uuid = state.getConfig().getClassificationUuid();
 448                         if (uuid == null) {
 449                                 String message = "No classification uuid is defined";
 450                                 fireWarningEvent(message, makeLocationStr(dataLocation), 6);
 451                                 result = createNewClassification(state);
 452                         } else {
 453                                 result = getClassificationService().find(uuid);
 454                                 if (result == null) {
 455                                         result = createNewClassification(state);
 456                                         result.setUuid(uuid);
 457                                 }
 458                         }
 459                         state.putTree(null, result);
 460                 }
 461                 save(result, state);
 462                 return result;
 463         }
 464
 465         private Classification createNewClassification(MarkupImportState state) {
 466                 Classification result;
 467                 result = Classification.NewInstance(state.getConfig().getClassificationTitle());
 468                 state.putTree(null, result);
 469                 return result;
 470         }
 471
 472         private Taxon handleTaxon(MarkupImportState state, XMLEventReader reader, StartElement parentEvent) throws XMLStreamException {
 473                 // TODO progress monitoring
 474                 Map<String, Attribute> attributes = getAttributes(parentEvent);
 475                 Taxon taxon = createTaxonAndName(state, attributes);
 476                 state.setCurrentTaxon(taxon);
 477
 478                 boolean hasTitle = false;
 479                 boolean hasNomenclature = false;
 480                 String taxonTitle = null;
 481
 482                 while (reader.hasNext()) {
 483                         XMLEvent next = readNoWhitespace(reader);
 484                         if (next.isEndElement()) {
 485                                 if (isMyEndingElement(next, parentEvent)) {
 486                                         checkMandatoryElement(hasTitle, parentEvent, TAXONTITLE);
 487                                         checkMandatoryElement(hasNomenclature, parentEvent,     NOMENCLATURE);
 488                                         handleUnexpectedAttributes(parentEvent.getLocation(),attributes);
 489                                         if (taxon.getName().getRank() == null){
 490                                                 String warning = "No rank exists for taxon " + taxon.getTitleCache();
 491                                                 fireWarningEvent(warning, next, 12);
 492                                         }
 493
 494                                         makeKeyNodes(state, parentEvent, taxonTitle);
 495                                         state.setCurrentTaxon(null);
 496                                         state.setCurrentTaxonNum(null);
 497                                         save(taxon, state);
 498                                         return taxon;
 499                                 } else {
 500                                         if (isEndingElement(next, HEADING)) {
 501                                                 // NOT YET IMPLEMENTED
 502                                                 popUnimplemented(next.asEndElement());
 503                                         } else if (isEndingElement(next, TEXT_SECTION)) {
 504                                                 // NOT YET IMPLEMENTED
 505                                                 popUnimplemented(next.asEndElement());
 506                                         } else if (isEndingElement(next, REFERENCES)) {
 507                                                 // NOT YET IMPLEMENTED
 508                                                 popUnimplemented(next.asEndElement());
 509                                         } else if (isEndingElement(next, FIGURE_REF)) {
 510                                                 // NOT YET IMPLEMENTED
 511                                                 popUnimplemented(next.asEndElement());
 512                                         } else {
 513                                                 handleUnexpectedEndElement(next.asEndElement());
 514                                         }
 515                                 }
 516                         } else if (next.isStartElement()) {
 517                                 if (isStartingElement(next, HEADING)) {
 518                                         handleNotYetImplementedElement(next);
 519                                 } else if (isStartingElement(next, TAXONTITLE)) {
 520                                         taxonTitle = handleTaxonTitle(state, reader, next);
 521                                         hasTitle = true;
 522                                 } else if (isStartingElement(next, WRITER)) {
 523                                         makeKeyWriter(state, reader, taxon, taxonTitle, next);
 524                                 } else if (isStartingElement(next, TEXT_SECTION)) {
 525                                         handleNotYetImplementedElement(next);
 526                                 } else if (isStartingElement(next, KEY)) {
 527                                         handleKey(state, reader, next);
 528                                 } else if (isStartingElement(next, NOMENCLATURE)) {
 529                                         handleNomenclature(state, reader, next);
 530                                         hasNomenclature = true;
 531                                 } else if (isStartingElement(next, FEATURE)) {
 532                                         handleFeature(state, reader, next);
 533                                 } else if (isStartingElement(next, NOTES)) {
 534                                         // TODO is this the correct way to handle notes?
 535                                         String note = handleNotes(state, reader, next);
 536
 537                                         UUID notesUuid;
 538                                         try {
 539                                                 notesUuid = state.getTransformer().getFeatureUuid(
 540                                                                 "notes");
 541                                                 Feature feature = getFeature(state, notesUuid, "Notes",
 542                                                                 "Notes", "note", null);
 543                                                 TextData textData = TextData.NewInstance(feature);
 544                                                 textData.putText(Language.DEFAULT(), note);
 545                                                 TaxonDescription description = getTaxonDescription(
 546                                                                 taxon, null, false, true);
 547                                                 description.addElement(textData);
 548                                         } catch (UndefinedTransformerMethodException e) {
 549                                                 String message = "getFeatureUuid method not yet implemented";
 550                                                 fireWarningEvent(message, next, 8);
 551                                         }
 552                                 } else if (isStartingElement(next, REFERENCES)) {
 553                                         handleNotYetImplementedElement(next);
 554                                 } else if (isStartingElement(next, FIGURE_REF)) {
 555                                         handleNotYetImplementedElement(next);
 556                                 } else if (isStartingElement(next, FIGURE)) {
 557                                         handleFigure(state, reader, next);
 558                                 } else if (isStartingElement(next, FOOTNOTE)) {
 559                                         FootnoteDataHolder footnote = handleFootnote(state, reader,     next);
 560                                         if (footnote.isRef()) {
 561                                                 String message = "Ref footnote not implemented here";
 562                                                 fireWarningEvent(message, next, 4);
 563                                         } else {
 564                                                 registerGivenFootnote(state, footnote);
 565                                         }
 566                                 } else {
 567                                         handleUnexpectedStartElement(next);
 568                                 }
 569                         } else {
 570                                 handleUnexpectedElement(next);
 571                         }
 572                 }
 573                 throw new IllegalStateException("<Taxon> has no closing tag");
 574         }
 575
 576         private void makeKeyNodes(MarkupImportState state, XMLEvent event, String taxonTitle) {
 577                 Taxon taxon = state.getCurrentTaxon();
 578                 String num = state.getCurrentTaxonNum();
 579
 580                 String nameString = CdmBase.deproxy(taxon.getName(), NonViralName.class).getNameCache();
 581 //              String nameString = taxonTitle;
 582
 583                 //try to find matching lead nodes
 584                 UnmatchedLeadsKey leadsKey = UnmatchedLeadsKey.NewInstance(num, nameString);
 585                 Set<PolytomousKeyNode> matchingNodes = handleMatchingNodes(state, taxon, leadsKey);
 586
 587                 if (num != null){//same without using the num
 588                         UnmatchedLeadsKey noNumLeadsKey = UnmatchedLeadsKey.NewInstance("", nameString);
 589                         Set<PolytomousKeyNode> noNumMatchingNodes = handleMatchingNodes(state, taxon, noNumLeadsKey);
 590                         if(noNumMatchingNodes.size() > 0){
 591                                 String message ="Taxon matches additional key node when not considering <num> attribute in taxontitle. This may be correct but may also indicate an error.";
 592                                 fireWarningEvent(message, event, 1);
 593                         }
 594                 }
 595                 //report missing match, if num exists
 596                 if (matchingNodes.isEmpty() && num != null){
 597                         String message = "Taxon has <num> attribute in taxontitle but no matching key nodes exist: %s, Key: %s";
 598                         message = String.format(message, num, leadsKey.toString());
 599                         fireWarningEvent(message, event, 1);
 600                 }
 601
 602         }
 603
 604         private Set<PolytomousKeyNode> handleMatchingNodes(MarkupImportState state, Taxon taxon, UnmatchedLeadsKey leadsKey) {
 605                 Set<PolytomousKeyNode> matchingNodes = state.getUnmatchedLeads().getNodes(leadsKey);
 606                 for (PolytomousKeyNode matchingNode : matchingNodes){
 607                         state.getUnmatchedLeads().removeNode(leadsKey, matchingNode);
 608                         matchingNode.setTaxon(taxon);
 609                         state.getPolytomousKeyNodesToSave().add(matchingNode);
 610                 }
 611                 return matchingNodes;
 612         }
 613
 614         private void handleKey(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
 615                 // attributes
 616                 Map<String, Attribute> attributes = getAttributes(parentEvent);
 617                 String isSpotcharacters = getAndRemoveAttributeValue(attributes, IS_SPOTCHARACTERS);
 618                 if (isNotBlank(isSpotcharacters) ) {
 619                         //TODO isSpotcharacters
 620                         String message = "Attribute isSpotcharacters not yet implemented for <key>";
 621                         fireWarningEvent(message, parentEvent, 4);
 622                 }
 623
 624                 PolytomousKey key = PolytomousKey.NewInstance();
 625                 key.addTaxonomicScope(state.getCurrentTaxon());
 626                 state.setCurrentKey(key);
 627
 628                 boolean isFirstCouplet = true;
 629                 while (reader.hasNext()) {
 630                         XMLEvent next = readNoWhitespace(reader);
 631                         if (isMyEndingElement(next, parentEvent)) {
 632                                 save(key, state);
 633                                 state.setCurrentKey(null);
 634                                 return;
 635                         } else if (isEndingElement(next, KEYNOTES)){
 636                                 popUnimplemented(next.asEndElement());
 637                         } else if (isStartingElement(next, KEY_TITLE)) {
 638                                 handleKeyTitle(state, reader, next);
 639                         } else if (isStartingElement(next, KEYNOTES)) {
 640                                 //TODO
 641                                 handleNotYetImplementedElement(next);
 642                         } else if (isStartingElement(next, COUPLET)) {
 643                                 PolytomousKeyNode node = null;
 644                                 if (isFirstCouplet){
 645                                         node = key.getRoot();
 646                                         isFirstCouplet = false;
 647                                 }
 648                                 handleCouplet(state, reader, next, node);
 649                         } else {
 650                                 handleUnexpectedElement(next);
 651                         }
 652                 }
 653                 throw new IllegalStateException("<key> has no closing tag");
 654         }
 655
 656         /**
 657          * @param state
 658          * @param reader
 659          * @param key
 660          * @param next
 661          * @throws XMLStreamException
 662          */
 663         private void handleKeyTitle(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
 664                 PolytomousKey key = state.getCurrentKey();
 665                 String keyTitle = getCData(state, reader, parentEvent);
 666                 String standardTitles = "(?i)(Key\\sto\\sthe\\s(genera|species|varieties|forms))";
 667
 668                 if (isNotBlank(keyTitle) ){
 669                         if (!state.getConfig().isReplaceStandardKeyTitles() || ! keyTitle.matches(standardTitles)){
 670                                 key.setTitleCache(keyTitle, true);
 671                         }
 672                 }
 673         }
 674
 675         private void handleCouplet(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent, PolytomousKeyNode parentNode) throws XMLStreamException {
 676                 String num = getOnlyAttribute(parentEvent, NUM, true);
 677                 List<PolytomousKeyNode> childList = new ArrayList<PolytomousKeyNode>();
 678
 679                 while (reader.hasNext()) {
 680                         XMLEvent next = readNoWhitespace(reader);
 681                         if (isMyEndingElement(next, parentEvent)) {
 682                                 completeCouplet(state, parentEvent, parentNode, num, childList);
 683                                 return;
 684                         } else if (isStartingElement(next, QUESTION)) {
 685                                 handleQuestion(state, reader, next, childList);
 686                         } else if (isStartingElement(next, KEYNOTES)) {
 687                                 //TODO
 688                                 handleNotYetImplementedElement(next);
 689                         } else if (isEndingElement(next, KEYNOTES)) {
 690                                 //TODO
 691                                 popUnimplemented(next.asEndElement());
 692                         } else {
 693                                 handleUnexpectedElement(next);
 694                         }
 695                 }
 696                 throw new IllegalStateException("<couplet> has no closing tag");
 697         }
 698
 699         /**
 700          * @param state
 701          * @param parentEvent
 702          * @param parentNode
 703          * @param num
 704          * @param childList
 705          */
 706         private void completeCouplet(MarkupImportState state, XMLEvent parentEvent,
 707                         PolytomousKeyNode parentNode, String num, List<PolytomousKeyNode> childList) {
 708                 if (parentNode != null){
 709                         for (PolytomousKeyNode childNode : childList){
 710                                 parentNode.addChild(childNode);
 711                         }
 712                 }else if (isNotBlank(num)){
 713                         UnmatchedLeadsKey unmatchedKey = UnmatchedLeadsKey.NewInstance(state.getCurrentKey(), num);
 714                         Set<PolytomousKeyNode> nodes = state.getUnmatchedLeads().getNodes(unmatchedKey);
 715                         for(PolytomousKeyNode nodeToMatch: nodes){
 716                                 for (PolytomousKeyNode childNode : childList){
 717                                         nodeToMatch.addChild(childNode);
 718                                 }
 719                                 state.getUnmatchedLeads().removeNode(unmatchedKey, nodeToMatch);
 720                         }
 721                 }else{
 722                         String message = "Parent num could not be matched. Please check if num (%s) is correct";
 723                         message = String.format(message, num);
 724                         fireWarningEvent(message, parentEvent, 6);
 725                 }
 726         }
 727
 728         private void handleQuestion(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent, List<PolytomousKeyNode> nodesList) throws XMLStreamException {
 729                 // attributes
 730                 Map<String, Attribute> attributes = getAttributes(parentEvent);
 731                 //needed only for data lineage
 732                 String questionNum = getAndRemoveRequiredAttributeValue(parentEvent, attributes, NUM);
 733
 734                 PolytomousKeyNode myNode = PolytomousKeyNode.NewInstance();
 735                 myNode.setKey(state.getCurrentKey());  //to avoid NPE while computing num in PolytomousKeyNode in case this node is not matched correctly with a parent
 736                 nodesList.add(myNode);
 737
 738                 while (reader.hasNext()) {
 739                         XMLEvent next = readNoWhitespace(reader);
 740                         if (isMyEndingElement(next, parentEvent)) {
 741                                 return;
 742                         } else if (isStartingElement(next, TEXT)) {
 743                                 String text = getCData(state, reader, next);
 744                                 KeyStatement statement = KeyStatement.NewInstance(text);
 745                                 myNode.setStatement(statement);
 746                         } else if (isStartingElement(next, COUPLET)) {
 747                                 //TODO test
 748                                 handleCouplet(state, reader, next, myNode);
 749                         } else if (isStartingElement(next, TO_COUPLET)) {
 750                                 handleToCouplet(state, reader, next, myNode);
 751                         } else if (isStartingElement(next, TO_TAXON)) {
 752                                 handleToTaxon(state, reader, next, myNode);
 753                         } else if (isStartingElement(next, TO_KEY)) {
 754                                 //TODO
 755                                 handleNotYetImplementedElement(next);
 756                         } else if (isEndingElement(next, TO_KEY)){
 757                                 //TODO
 758                                 popUnimplemented(next.asEndElement());
 759                         } else if (isStartingElement(next, KEYNOTES)) {
 760                                 //TODO
 761                                 handleNotYetImplementedElement(next);
 762                         } else if (isEndingElement(next, KEYNOTES)){
 763                                 //TODO
 764                                 popUnimplemented(next.asEndElement());
 765                         } else {
 766                                 handleUnexpectedElement(next);
 767                         }
 768                 }
 769                 throw new IllegalStateException("<question> has no closing tag");
 770         }
 771
 772         private void handleToCouplet(MarkupImportState state, XMLEventReader reader, XMLEvent next, PolytomousKeyNode node) throws XMLStreamException {
 773                 String num = getOnlyAttribute(next, NUM, true);
 774                 String cData = getCData(state, reader, next, false);
 775                 if (isNotBlank(cData) && ! cData.equals(num)){
 776                         String message = "CData ('%s') not handled in <toCouplet>";
 777                         message = String.format(message, cData);
 778                         fireWarningEvent(message, next, 4);
 779                 }
 780                 UnmatchedLeadsKey unmatched = UnmatchedLeadsKey.NewInstance(state.getCurrentKey(), num);
 781                 state.getUnmatchedLeads().addKey(unmatched, node);
 782         }
 783
 784         private void handleToTaxon(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent, PolytomousKeyNode node) throws XMLStreamException {
 785                 Map<String, Attribute> attributes = getAttributes(parentEvent);
 786                 String num = getAndRemoveAttributeValue(attributes, NUM);
 787                 String taxonStr = getCData(state, reader, parentEvent, false);
 788                 //TODO ?
 789                 taxonStr = makeTaxonKey(taxonStr, state.getCurrentTaxon());
 790                 UnmatchedLeadsKey unmatched = UnmatchedLeadsKey.NewInstance(num, taxonStr);
 791                 state.getUnmatchedLeads().addKey(unmatched, node);
 792                 return;
 793         }
 794
 795         private String makeTaxonKey(String strGoto, Taxon taxon) {
 796                 String result = "";
 797                 if (strGoto == null){
 798                         return "";
 799                 }
 800
 801                 NonViralName<?> name = CdmBase.deproxy(taxon.getName(), NonViralName.class);
 802                 String strGenusName = name.getGenusOrUninomial();
 803
 804
 805                 strGoto = strGoto.replaceAll("\\([^\\(\\)]*\\)", "");  //replace all brackets
 806                 strGoto = strGoto.replaceAll("\\s+", " "); //replace multiple whitespaces by exactly one whitespace
 807
 808                 strGoto = strGoto.trim();
 809                 String[] split = strGoto.split("\\s");
 810                 for (int i = 0; i<split.length; i++){
 811                         String single = split[i];
 812                         if (isGenusAbbrev(single, strGenusName)){
 813                                 split[i] = strGenusName;
 814                         }
 815                         if (isInfraSpecificMarker(single)){
 816                                 String strSpeciesEpi = name.getSpecificEpithet();
 817                                 if (isBlank(result)){
 818                                         result += strGenusName + " " + strSpeciesEpi;
 819                                 }
 820                         }
 821                         result = (result + " " + split[i]).trim();
 822                 }
 823                 return result;
 824         }
 825
 826
 827         private boolean isInfraSpecificMarker(String single) {
 828                 try {
 829                         if (Rank.getRankByAbbreviation(single).isInfraSpecific()){
 830                                 return true;
 831                         }else{
 832                                 return false;
 833                         }
 834                 } catch (UnknownCdmTypeException e) {
 835                         return false;
 836                 }
 837         }
 838
 839         private boolean isGenusAbbrev(String single, String strGenusName) {
 840                 if (! single.matches("[A-Z]\\.?")) {
 841                         return false;
 842                 }else if (single.length() == 0 || strGenusName == null || strGenusName.length() == 0){
 843                         return false;
 844                 }else{
 845                         return single.charAt(0) == strGenusName.charAt(0);
 846                 }
 847         }
 848
 849         /**
 850          * @param state
 851          * @param reader
 852          * @param taxon
 853          * @param taxonTitle
 854          * @param next
 855          * @throws XMLStreamException
 856          */
 857         private void makeKeyWriter(MarkupImportState state, XMLEventReader reader, Taxon taxon, String taxonTitle, XMLEvent next) throws XMLStreamException {
 858                 WriterDataHolder writer = handleWriter(state, reader, next);
 859                 taxon.addExtension(writer.extension);
 860                 // TODO what if taxonTitle comes later
 861                 if (StringUtils.isNotBlank(taxonTitle)
 862                                 && writer.extension != null) {
 863                         Reference<?> sec = ReferenceFactory.newBookSection();
 864                         sec.setTitle(taxonTitle);
 865                         TeamOrPersonBase<?> author = createAuthor(writer.writer);
 866                         sec.setAuthorTeam(author);
 867                         sec.setInReference(state.getConfig()
 868                                         .getSourceReference());
 869                         taxon.setSec(sec);
 870                         registerFootnotes(state, sec, writer.footnotes);
 871                 } else {
 872                         String message = "No taxontitle exists for writer";
 873                         fireWarningEvent(message, next, 6);
 874                 }
 875         }
 876
 877         private String handleNotes(MarkupImportState state, XMLEventReader reader,
 878                         XMLEvent parentEvent) throws XMLStreamException {
 879                 checkNoAttributes(parentEvent);
 880
 881                 String text = "";
 882                 while (reader.hasNext()) {
 883                         XMLEvent next = readNoWhitespace(reader);
 884                         if (isMyEndingElement(next, parentEvent)) {
 885                                 return text;
 886                         } else if (next.isEndElement()) {
 887                                 if (isEndingElement(next, HEADING)) {
 888                                         popUnimplemented(next.asEndElement());
 889                                 } else if (isEndingElement(next, WRITER)) {
 890                                         popUnimplemented(next.asEndElement());
 891                                 } else if (isEndingElement(next, NUM)) {
 892                                         popUnimplemented(next.asEndElement());
 893                                 } else {
 894                                         handleUnexpectedEndElement(next.asEndElement());
 895                                 }
 896                         } else if (next.isStartElement()) {
 897                                 if (isStartingElement(next, HEADING)) {
 898                                         handleNotYetImplementedElement(next);
 899                                 } else if (isStartingElement(next, SUB_HEADING)) {
 900                                         String subheading = getCData(state, reader, next).trim();
 901                                         if (! isNoteHeading(subheading)) {
 902                                                 fireNotYetImplementedElement(next.getLocation(), next.asStartElement().getName(), 0);
 903                                         }
 904                                 } else if (isStartingElement(next, WRITER)) {
 905                                         handleNotYetImplementedElement(next);
 906                                 } else if (isStartingElement(next, NUM)) {
 907                                         handleNotYetImplementedElement(next);
 908                                 } else if (isStartingElement(next, STRING)) {
 909                                         // TODO why multiple strings in schema?
 910                                         text = makeNotesString(state, reader, text, next);
 911                                 } else {
 912                                         handleUnexpectedStartElement(next.asStartElement());
 913                                 }
 914                         } else {
 915                                 handleUnexpectedElement(next);
 916                         }
 917                 }
 918                 throw new IllegalStateException("<Notes> has no closing tag");
 919         }
 920
 921         /**
 922          * @param state
 923          * @param reader
 924          * @param text
 925          * @param next
 926          * @return
 927          * @throws XMLStreamException
 928          */
 929         private String makeNotesString(MarkupImportState state, XMLEventReader reader, String text, XMLEvent next) throws XMLStreamException {
 930                 Map<String, String> stringMap = handleString(state, reader,     next, null);
 931                 if (stringMap.size() == 0){
 932                         String message = "No text available in <notes>";
 933                         fireWarningEvent(message, next, 4);
 934                 }else if (stringMap.size() > 1){
 935                         String message = "Subheadings not yet supported in <notes>";
 936                         fireWarningEvent(message, next, 4);
 937                 }else{
 938                         String firstSubheading = stringMap.keySet().iterator().next();
 939                         if ( firstSubheading != null && ! isNoteHeading (firstSubheading) )  {
 940                                 String message = "Subheadings not yet supported in <notes>";
 941                                 fireWarningEvent(message, next, 4);
 942                         }
 943                 }
 944                 for (String subheading : stringMap.keySet()){
 945                         text += subheading;
 946                         text += stringMap.get(subheading);
 947                 }
 948                 return text;
 949         }
 950
 951         private boolean isNoteHeading(String heading) {
 952                 String excludePattern = "(i?)(Notes?):?";
 953                 return heading.matches(excludePattern);
 954         }
 955
 956         /**
 957          * @param state
 958          * @param attributes
 959          */
 960         private Taxon createTaxonAndName(MarkupImportState state,
 961                         Map<String, Attribute> attributes) {
 962                 NonViralName<?> name;
 963                 Rank rank = null;  //Rank.SPECIES(); // default
 964                 boolean isCultivar = checkAndRemoveAttributeValue(attributes, CLASS, "cultivated");
 965                 if (isCultivar) {
 966                         name = CultivarPlantName.NewInstance(rank);
 967                 } else {
 968                         name = createNameByCode(state, rank);
 969                 }
 970                 Taxon taxon = Taxon.NewInstance(name, state.getConfig().getSourceReference());
 971                 if (checkAndRemoveAttributeValue(attributes, CLASS, "dubious")) {
 972                         taxon.setDoubtful(true);
 973                 } else if (checkAndRemoveAttributeValue(attributes, CLASS, "excluded")) {
 974                         taxon.setExcluded(true);
 975                 }
 976                 // TODO insufficient, new, expected
 977                 handleNotYetImplementedAttribute(attributes, CLASS);
 978                 // From old version
 979                 // MarkerType markerType = getMarkerType(state, attrValue);
 980                 // if (markerType == null){
 981                 // logger.warn("Class attribute value for taxon not yet supported: " +
 982                 // attrValue);
 983                 // }else{
 984                 // taxon.addMarker(Marker.NewInstance(markerType, true));
 985                 // }
 986
 987                 // save(name, state);
 988                 // save(taxon, state);
 989                 return taxon;
 990         }
 991
 992         /**
 993          * @param state
 994          * @param rank
 995          * @return
 996          */
 997         private NonViralName<?> createNameByCode(MarkupImportState state, Rank rank) {
 998                 NonViralName<?> name;
 999                 NomenclaturalCode nc = makeNomenclaturalCode(state);
1000                 name = (NonViralName<?>) nc.getNewTaxonNameInstance(rank);
1001                 return name;
1002         }
1003
1004         /**
1005          * @param state
1006          * @return
1007          */
1008         private NomenclaturalCode makeNomenclaturalCode(MarkupImportState state) {
1009                 NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
1010                 if (nc == null) {
1011                         nc = NomenclaturalCode.ICBN; // default;
1012                 }
1013                 return nc;
1014         }
1015
1016         private String handleTaxonTitle(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1017                 //attributes
1018                 String text = "";
1019                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1020                 String rankAttr = getAndRemoveAttributeValue(attributes, RANK);
1021                 Rank rank = makeRank(state, rankAttr, false);
1022                 String num = getAndRemoveAttributeValue(attributes, NUM);
1023                 state.setCurrentTaxonNum(num);
1024                 checkNoAttributes(attributes, parentEvent);
1025
1026                 // TODO handle attributes
1027                 while (reader.hasNext()) {
1028                         XMLEvent next = readNoWhitespace(reader);
1029                         if (next.isEndElement()) {
1030                                 if (isMyEndingElement(next, parentEvent)) {
1031                                         Taxon taxon = state.getCurrentTaxon();
1032                                         String titleText = null;
1033                                         if (checkMandatoryText(text, parentEvent)) {
1034                                                 titleText = normalize(text);
1035                                                 UUID uuidTitle = MarkupTransformer.uuidTaxonTitle;
1036                                                 ExtensionType titleExtension = this.getExtensionType(state, uuidTitle, "Taxon Title ","taxon title", "title");
1037                                                 taxon.addExtension(titleText, titleExtension);
1038                                         }
1039                                         taxon.getName().setRank(rank);
1040                                         // TODO check title exists
1041                                         return titleText;
1042                                 } else {
1043                                         if (isEndingElement(next, FOOTNOTE)) {
1044                                                 // NOT YET IMPLEMENTED
1045                                                 popUnimplemented(next.asEndElement());
1046                                         } else {
1047                                                 handleUnexpectedEndElement(next.asEndElement());
1048                                                 state.setUnsuccessfull();
1049                                         }
1050                                 }
1051                         } else if (next.isStartElement()) {
1052                                 if (isStartingElement(next, FOOTNOTE)) {
1053                                         handleNotYetImplementedElement(next);
1054                                 } else {
1055                                         handleUnexpectedStartElement(next);
1056                                         state.setUnsuccessfull();
1057                                 }
1058                         } else if (next.isCharacters()) {
1059                                 text += next.asCharacters().getData();
1060
1061                         } else {
1062                                 handleUnexpectedElement(next);
1063                                 state.setUnsuccessfull();
1064                         }
1065                 }
1066                 return null;
1067
1068         }
1069
1070         private WriterDataHolder handleWriter(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1071                 String text = "";
1072                 checkNoAttributes(parentEvent);
1073                 WriterDataHolder dataHolder = new WriterDataHolder();
1074                 List<FootnoteDataHolder> footnotes = new ArrayList<FootnoteDataHolder>();
1075
1076                 // TODO handle attributes
1077                 while (reader.hasNext()) {
1078                         XMLEvent next = readNoWhitespace(reader);
1079                         if (isMyEndingElement(next, parentEvent)) {
1080                                 text = CdmUtils.removeBrackets(text);
1081                                 if (checkMandatoryText(text, parentEvent)) {
1082                                         text = normalize(text);
1083                                         dataHolder.writer = text;
1084                                         dataHolder.footnotes = footnotes;
1085
1086                                         // Extension
1087                                         UUID uuidWriterExtension = MarkupTransformer.uuidWriterExtension;
1088                                         ExtensionType writerExtensionType = this
1089                                                         .getExtensionType(state, uuidWriterExtension,
1090                                                                         "Writer", "writer", "writer");
1091                                         Extension extension = Extension.NewInstance();
1092                                         extension.setType(writerExtensionType);
1093                                         extension.setValue(text);
1094                                         dataHolder.extension = extension;
1095
1096                                         // Annotation
1097                                         UUID uuidWriterAnnotation = MarkupTransformer.uuidWriterAnnotation;
1098                                         AnnotationType writerAnnotationType = this.getAnnotationType(state, uuidWriterAnnotation, "Writer", "writer", "writer", null);
1099                                         Annotation annotation = Annotation.NewInstance(text, writerAnnotationType, Language.DEFAULT());
1100                                         dataHolder.annotation = annotation;
1101
1102                                         return dataHolder;
1103                                 } else {
1104                                         return null;
1105                                 }
1106                         } else if (isStartingElement(next, FOOTNOTE_REF)) {
1107                                 FootnoteDataHolder footNote = handleFootnoteRef(state, reader, next);
1108                                 if (footNote.isRef()) {
1109                                         footnotes.add(footNote);
1110                                 } else {
1111                                         logger.warn("Non ref footnotes not yet impelemnted");
1112                                 }
1113                         } else if (next.isCharacters()) {
1114                                 text += next.asCharacters().getData();
1115
1116                         } else {
1117                                 handleUnexpectedElement(next);
1118                                 state.setUnsuccessfull();
1119                         }
1120                 }
1121                 throw new IllegalStateException("<writer> has no end tag");
1122         }
1123
1124         private void registerFootnotes(MarkupImportState state, AnnotatableEntity entity, List<FootnoteDataHolder> footnotes) {
1125                 for (FootnoteDataHolder footNote : footnotes) {
1126                         registerFootnoteDemand(state, entity, footNote);
1127                 }
1128         }
1129
1130         private void registerGivenFootnote(MarkupImportState state, FootnoteDataHolder footnote) {
1131                 state.registerFootnote(footnote);
1132                 Set<AnnotatableEntity> demands = state.getFootnoteDemands(footnote.id);
1133                 if (demands != null) {
1134                         for (AnnotatableEntity entity : demands) {
1135                                 attachFootnote(state, entity, footnote);
1136                         }
1137                 }
1138         }
1139
1140         private void registerGivenFigure(MarkupImportState state, String id, Media figure) {
1141                 state.registerFigure(id, figure);
1142                 Set<AnnotatableEntity> demands = state.getFigureDemands(id);
1143                 if (demands != null) {
1144                         for (AnnotatableEntity entity : demands) {
1145                                 attachFigure(state, entity, figure);
1146                         }
1147                 }
1148         }
1149
1150         private void registerFootnoteDemand(MarkupImportState state, AnnotatableEntity entity, FootnoteDataHolder footnote) {
1151                 FootnoteDataHolder existingFootnote = state.getFootnote(footnote.ref);
1152                 if (existingFootnote != null) {
1153                         attachFootnote(state, entity, existingFootnote);
1154                 } else {
1155                         Set<AnnotatableEntity> demands = state.getFootnoteDemands(footnote.ref);
1156                         if (demands == null) {
1157                                 demands = new HashSet<AnnotatableEntity>();
1158                                 state.putFootnoteDemands(footnote.ref, demands);
1159                         }
1160                         demands.add(entity);
1161                 }
1162         }
1163
1164         private void registerFigureDemand(MarkupImportState state, AnnotatableEntity entity, String figureRef) {
1165                 Media existingFigure = state.getFigure(figureRef);
1166                 if (existingFigure != null) {
1167                         attachFigure(state, entity, existingFigure);
1168                 } else {
1169                         Set<AnnotatableEntity> demands = state.getFigureDemands(figureRef);
1170                         if (demands == null) {
1171                                 demands = new HashSet<AnnotatableEntity>();
1172                                 state.putFigureDemands(figureRef, demands);
1173                         }
1174                         demands.add(entity);
1175                 }
1176         }
1177
1178         private void attachFootnote(MarkupImportState state, AnnotatableEntity entity, FootnoteDataHolder footnote) {
1179                 AnnotationType annotationType = this.getAnnotationType(state, MarkupTransformer.uuidFootnote, "Footnote", "An e-flora footnote", "fn", null);
1180                 Annotation annotation = Annotation.NewInstance(footnote.string,
1181                                 annotationType, Language.DEFAULT());
1182                 // TODO transient objects
1183                 entity.addAnnotation(annotation);
1184                 save(entity, state);
1185         }
1186
1187         private void attachFigure(MarkupImportState state,
1188                         AnnotatableEntity entity, Media figure) {
1189                 // IdentifiableEntity<?> toSave;
1190                 if (entity.isInstanceOf(TextData.class)) {
1191                         TextData deb = CdmBase.deproxy(entity, TextData.class);
1192                         deb.addMedia(figure);
1193                         // toSave = ((TaxonDescription)deb.getInDescription()).getTaxon();
1194                 } else if (entity.isInstanceOf(IdentifiableMediaEntity.class)) {
1195                         IdentifiableMediaEntity<?> ime = CdmBase.deproxy(entity,
1196                                         IdentifiableMediaEntity.class);
1197                         ime.addMedia(figure);
1198                         // toSave = ime;
1199                 } else {
1200                         String message = "Unsupported entity to attach media: %s";
1201                         message = String.format(message, entity.getClass().getName());
1202                         // toSave = null;
1203                 }
1204                 save(entity, state);
1205         }
1206
1207         private void handleFigure(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1208                 // FigureDataHolder result = new FigureDataHolder();
1209
1210                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1211                 String id = getAndRemoveAttributeValue(attributes, ID);
1212                 String type = getAndRemoveAttributeValue(attributes, TYPE);
1213                 checkNoAttributes(attributes, parentEvent);
1214
1215                 String urlString = null;
1216                 String legendString = null;
1217                 String titleString = null;
1218                 String numString = null;
1219                 String text = null;
1220                 while (reader.hasNext()) {
1221                         XMLEvent next = readNoWhitespace(reader);
1222                         if (isMyEndingElement(next, parentEvent)) {
1223                                 makeFigure(state, id, type, urlString, legendString, titleString, numString, next);
1224                                 return;
1225                         } else if (isStartingElement(next, FIGURE_LEGEND)) {
1226                                 // TODO same as figurestring ?
1227                                 legendString = handleFootnoteString(state, reader, next);
1228                         } else if (isStartingElement(next, FIGURE_TITLE)) {
1229                                 titleString = getCData(state, reader, next);
1230                         } else if (isStartingElement(next, URL)) {
1231                                 String localUrl = getCData(state, reader, next);
1232                                 urlString = CdmUtils.Nz(state.getBaseMediaUrl()) + localUrl;
1233                         } else if (isStartingElement(next, NUM)) {
1234                                 numString = getCData(state, reader, next);
1235                         } else if (next.isCharacters()) {
1236                                 text += next.asCharacters().getData();
1237                         } else {
1238                                 fireUnexpectedEvent(next, 0);
1239                         }
1240                 }
1241                 throw new IllegalStateException("<figure> has no end tag");
1242         }
1243
1244         /**
1245          * @param state
1246          * @param id
1247          * @param type
1248          * @param urlString
1249          * @param legendString
1250          * @param titleString
1251          * @param numString
1252          * @param next
1253          */
1254         private void makeFigure(MarkupImportState state, String id, String type, String urlString,
1255                                                 String legendString, String titleString, String numString, XMLEvent next) {
1256                 Media media = null;
1257                 boolean isFigure = false;
1258                 try {
1259                         //TODO maybe everything is a figure as it is all taken from a book
1260                         if ("lineart".equals(type)) {
1261                                 isFigure = true;
1262 //                              media = Figure.NewInstance(url.toURI(), null, null,     null);
1263                         } else if (type == null || "photo".equals(type)
1264                                         || "signature".equals(type)
1265                                         || "others".equals(type)) {
1266                         } else {
1267                                 String message = "Unknown figure type '%s'";
1268                                 message = String.format(message, type);
1269                                 fireWarningEvent(message, next, 2);
1270                         }
1271                         media = docImport.getImageMedia(urlString, docImport.getReadMediaData(), isFigure);
1272
1273                         if (media != null){
1274                                 // title
1275                                 if (StringUtils.isNotBlank(titleString)) {
1276                                         media.putTitle(Language.DEFAULT(), titleString);
1277                                 }
1278                                 // legend
1279                                 if (StringUtils.isNotBlank(legendString)) {
1280                                         media.addDescription(legendString, Language.DEFAULT());
1281                                 }
1282                                 if (StringUtils.isNotBlank(numString)) {
1283                                         // TODO use concrete source (e.g. DAPHNIPHYLLACEAE in FM
1284                                         // vol.13)
1285                                         Reference<?> citation = state.getConfig().getSourceReference();
1286                                         media.addSource(numString, "num", citation, null);
1287                                         // TODO name used in source if available
1288                                 }
1289                                 // TODO which citation
1290                                 if (StringUtils.isNotBlank(id)) {
1291                                         media.addSource(id, null, state.getConfig().getSourceReference(), null);
1292                                 } else {
1293                                         String message = "Figure id should never be empty or null";
1294                                         fireWarningEvent(message, next, 6);
1295                                 }
1296
1297                                 // text
1298                                 // do nothing
1299
1300                         }
1301                 } catch (MalformedURLException e) {
1302                         String message = "Media uri has incorrect syntax: %s";
1303                         message = String.format(message, urlString);
1304                         fireWarningEvent(message, next, 4);
1305 //              } catch (URISyntaxException e) {
1306 //                      String message = "Media uri has incorrect syntax: %s";
1307 //                      message = String.format(message, urlString);
1308 //                      fireWarningEvent(message, next, 4);
1309                 }
1310
1311                 registerGivenFigure(state, id, media);
1312         }
1313
1314         private FigureDataHolder handleFigureRef(MarkupImportState state,
1315                         XMLEventReader reader, XMLEvent parentEvent)
1316                         throws XMLStreamException {
1317                 FigureDataHolder result = new FigureDataHolder();
1318                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1319                 result.ref = getAndRemoveAttributeValue(attributes, REF);
1320                 checkNoAttributes(attributes, parentEvent);
1321
1322                 // text is not handled, needed only for debugging purposes
1323                 String text = "";
1324                 while (reader.hasNext()) {
1325                         XMLEvent next = readNoWhitespace(reader);
1326                         if (isMyEndingElement(next, parentEvent)) {
1327                                 return result;
1328                         } else if (isStartingElement(next, NUM)) {
1329                                 String num = getCData(state, reader, next);
1330                                 result.num = num; // num is not handled during import
1331                         } else if (isStartingElement(next, FIGURE_PART)) {
1332                                 result.figurePart = getCData(state, reader, next);
1333                         } else if (next.isCharacters()) {
1334                                 text += next.asCharacters().getData();
1335                         } else {
1336                                 fireUnexpectedEvent(next, 0);
1337                         }
1338                 }
1339                 throw new IllegalStateException("<figureRef> has no end tag");
1340         }
1341
1342         private FootnoteDataHolder handleFootnote(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1343                 FootnoteDataHolder result = new FootnoteDataHolder();
1344                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1345                 result.id = getAndRemoveAttributeValue(attributes, ID);
1346                 // result.ref = getAndRemoveAttributeValue(attributes, REF);
1347                 checkNoAttributes(attributes, parentEvent);
1348
1349                 while (reader.hasNext()) {
1350                         XMLEvent next = readNoWhitespace(reader);
1351                         if (isStartingElement(next, FOOTNOTE_STRING)) {
1352                                 String string = handleFootnoteString(state, reader, next);
1353                                 result.string = string;
1354                         } else if (isMyEndingElement(next, parentEvent)) {
1355                                 return result;
1356                         } else {
1357                                 fireUnexpectedEvent(next, 0);
1358                         }
1359                 }
1360                 return result;
1361         }
1362
1363         private FootnoteDataHolder handleFootnoteRef(MarkupImportState state,
1364                         XMLEventReader reader, XMLEvent parentEvent)
1365                         throws XMLStreamException {
1366                 FootnoteDataHolder result = new FootnoteDataHolder();
1367                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1368                 result.ref = getAndRemoveAttributeValue(attributes, REF);
1369                 checkNoAttributes(attributes, parentEvent);
1370
1371                 // text is not handled, needed only for debugging purposes
1372                 String text = "";
1373                 while (reader.hasNext()) {
1374                         XMLEvent next = readNoWhitespace(reader);
1375                         // if (isStartingElement(next, FOOTNOTE_STRING)){
1376                         // String string = handleFootnoteString(state, reader, next);
1377                         // result.string = string;
1378                         // }else
1379                         if (isMyEndingElement(next, parentEvent)) {
1380                                 return result;
1381                         } else if (next.isCharacters()) {
1382                                 text += next.asCharacters().getData();
1383
1384                         } else {
1385                                 fireUnexpectedEvent(next, 0);
1386                         }
1387                 }
1388                 return result;
1389         }
1390
1391         private void handleNomenclature(MarkupImportState state,
1392                         XMLEventReader reader, XMLEvent parentEvent)
1393                         throws XMLStreamException {
1394                 checkNoAttributes(parentEvent);
1395
1396                 while (reader.hasNext()) {
1397                         XMLEvent next = readNoWhitespace(reader);
1398                         if (isStartingElement(next, HOMOTYPES)) {
1399                                 handleHomotypes(state, reader, next.asStartElement());
1400                         } else if (isMyEndingElement(next, parentEvent)) {
1401                                 return;
1402                         } else {
1403                                 fireSchemaConflictEventExpectedStartTag(HOMOTYPES, reader);
1404                                 state.setUnsuccessfull();
1405                         }
1406                 }
1407                 return;
1408         }
1409
1410         private String handleFootnoteString(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1411                 boolean isTextMode = true;
1412                 String text = "";
1413                 while (reader.hasNext()) {
1414                         XMLEvent next = readNoWhitespace(reader);
1415                         if (isMyEndingElement(next, parentEvent)) {
1416                                 return text;
1417                         } else if (next.isEndElement()) {
1418                                 if (isEndingElement(next, FULL_NAME)) {
1419                                         popUnimplemented(next.asEndElement());
1420                                 } else if (isEndingElement(next, BR)) {
1421                                         isTextMode = true;
1422                                 } else if (isHtml(next)) {
1423                                         text += getXmlTag(next);
1424                                 } else {
1425                                         handleUnexpectedEndElement(next.asEndElement());
1426                                 }
1427                         } else if (next.isStartElement()) {
1428                                 if (isStartingElement(next, FULL_NAME)) {
1429                                         handleNotYetImplementedElement(next);
1430                                 } else if (isStartingElement(next, GATHERING)) {
1431                                         text += handleInLineGathering(state, reader, next);
1432                                 } else if (isStartingElement(next, REFERENCES)) {
1433                                         text += " " + handleInLineReferences(state, reader, next)+ " ";
1434                                 } else if (isStartingElement(next, BR)) {
1435                                         text += "<br/>";
1436                                         isTextMode = false;
1437                                 } else if (isHtml(next)) {
1438                                         text += getXmlTag(next);
1439                                 } else {
1440                                         handleUnexpectedStartElement(next.asStartElement());
1441                                 }
1442                         } else if (next.isCharacters()) {
1443                                 if (!isTextMode) {
1444                                         String message = "footnoteString is not in text mode";
1445                                         fireWarningEvent(message, next, 6);
1446                                 } else {
1447                                         text += next.asCharacters().getData().trim();
1448                                         // getCData(state, reader, next); does not work as we have inner tags like <references>
1449                                 }
1450                         } else {
1451                                 handleUnexpectedEndElement(next.asEndElement());
1452                         }
1453                 }
1454                 throw new IllegalStateException("<footnoteString> has no closing tag");
1455
1456         }
1457
1458         private String handleInLineGathering(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1459                 DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(DerivedUnitType.DerivedUnit.FieldObservation);
1460                 handleGathering(state, reader, parentEvent, facade);
1461                 FieldObservation fieldObservation = facade.innerFieldObservation();
1462                 String result = "<cdm:specimen uuid='%s'>%s</specimen>";
1463                 result = String.format(result, fieldObservation.getUuid(), fieldObservation.getTitleCache());
1464                 save(fieldObservation, state);
1465                 return result;
1466         }
1467
1468         private String handleInLineReferences(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1469                 checkNoAttributes(parentEvent);
1470
1471                 boolean hasReference = false;
1472                 String text = "";
1473                 while (reader.hasNext()) {
1474                         XMLEvent next = readNoWhitespace(reader);
1475                         if (isMyEndingElement(next, parentEvent)) {
1476                                 checkMandatoryElement(hasReference, parentEvent.asStartElement(), REFERENCE);
1477                                 return text;
1478                         } else if (isStartingElement(next, REFERENCE)) {
1479                                 text += handleInLineReference(state, reader, next);
1480                                 hasReference = true;
1481                         } else {
1482                                 handleUnexpectedElement(next);
1483                         }
1484                 }
1485                 throw new IllegalStateException("<References> has no closing tag");
1486         }
1487
1488         private String handleInLineReference(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent)throws XMLStreamException {
1489                 Reference<?> reference = handleReference(state, reader, parentEvent);
1490                 String result = "<cdm:ref uuid='%s'>%s</ref>";
1491                 result = String.format(result, reference.getUuid(), reference.getTitleCache());
1492                 save(reference, state);
1493                 return result;
1494         }
1495
1496         private Reference<?> handleReference(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent)throws XMLStreamException {
1497                 checkNoAttributes(parentEvent);
1498
1499                 boolean hasRefPart = false;
1500                 Map<String, String> refMap = new HashMap<String, String>();
1501                 while (reader.hasNext()) {
1502                         XMLEvent next = readNoWhitespace(reader);
1503                         if (isMyEndingElement(next, parentEvent)) {
1504                                 checkMandatoryElement(hasRefPart, parentEvent.asStartElement(),
1505                                                 REF_PART);
1506                                 Reference<?> reference = createReference(state, refMap, next);
1507                                 return reference;
1508                         } else if (isStartingElement(next, REF_PART)) {
1509                                 handleRefPart(state, reader, next, refMap);
1510                                 hasRefPart = true;
1511                         } else {
1512                                 handleUnexpectedElement(next);
1513                         }
1514                 }
1515                 // TODO handle missing end element
1516                 throw new IllegalStateException("<Reference> has no closing tag");
1517         }
1518
1519         private void handleHomotypes(MarkupImportState state, XMLEventReader reader, StartElement parentEvent)
1520                         throws XMLStreamException {
1521                 checkNoAttributes(parentEvent);
1522
1523                 HomotypicalGroup homotypicalGroup = null;
1524
1525                 boolean hasNom = false;
1526                 while (reader.hasNext()) {
1527                         XMLEvent next = readNoWhitespace(reader);
1528                         if (next.isEndElement()) {
1529                                 if (isMyEndingElement(next, parentEvent)) {
1530                                         checkMandatoryElement(hasNom, parentEvent, NOM);
1531                                         return;
1532                                 } else {
1533                                         if (isEndingElement(next, NAME_TYPE)) {
1534                                                 state.setNameType(false);
1535                                         } else if (isEndingElement(next, NOTES)) {
1536                                                 // NOT YET IMPLEMENTED
1537                                                 popUnimplemented(next.asEndElement());
1538                                         } else {
1539                                                 handleUnexpectedEndElement(next.asEndElement());
1540                                         }
1541                                 }
1542                         } else if (next.isStartElement()) {
1543                                 if (isStartingElement(next, NOM)) {
1544                                         NonViralName<?> name = handleNom(state, reader, next, homotypicalGroup);
1545                                         homotypicalGroup = name.getHomotypicalGroup();
1546                                         hasNom = true;
1547                                 } else if (isStartingElement(next, NAME_TYPE)) {
1548                                         state.setNameType(true);
1549                                         handleNameType(state, reader, next, homotypicalGroup);
1550                                 } else if (isStartingElement(next, SPECIMEN_TYPE)) {
1551                                         handleSpecimenType(state, reader, next, homotypicalGroup);
1552                                 } else if (isStartingElement(next, NOTES)) {
1553                                         handleNotYetImplementedElement(next);
1554                                 } else {
1555                                         handleUnexpectedStartElement(next);
1556                                 }
1557                         } else {
1558                                 handleUnexpectedElement(next);
1559                         }
1560                 }
1561                 // TODO handle missing end element
1562                 throw new IllegalStateException("Homotypes has no closing tag");
1563
1564         }
1565
1566         private void handleNameType(MarkupImportState state, XMLEventReader reader,
1567                         XMLEvent parentEvent, HomotypicalGroup homotypicalGroup)
1568                         throws XMLStreamException {
1569                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1570                 String typeStatus = getAndRemoveAttributeValue(attributes, TYPE_STATUS);
1571                 checkNoAttributes(attributes, parentEvent);
1572
1573                 NameTypeDesignationStatus status;
1574                 try {
1575                         status = NameTypeParser.parseNameTypeStatus(typeStatus);
1576                 } catch (UnknownCdmTypeException e) {
1577                         String message = "Type status could not be recognized: %s";
1578                         message = String.format(message, typeStatus);
1579                         fireWarningEvent(message, parentEvent, 4);
1580                         status = null;
1581                 }
1582
1583                 boolean hasNom = false;
1584                 while (reader.hasNext()) {
1585                         XMLEvent next = readNoWhitespace(reader);
1586                         if (next.isEndElement()) {
1587                                 if (isMyEndingElement(next, parentEvent)) {
1588                                         checkMandatoryElement(hasNom, parentEvent.asStartElement(),
1589                                                         NOM);
1590                                         state.setNameType(false);
1591                                         return;
1592                                 } else {
1593                                         if (isEndingElement(next, ACCEPTED_NAME)) {
1594                                                 // NOT YET IMPLEMENTED
1595                                                 popUnimplemented(next.asEndElement());
1596                                         } else {
1597                                                 handleUnexpectedEndElement(next.asEndElement());
1598                                         }
1599                                 }
1600                         } else if (next.isStartElement()) {
1601                                 if (isStartingElement(next, NOM)) {
1602                                         // TODO should we check if the type is always a species, is
1603                                         // this a rule?
1604                                         NonViralName<?> speciesName = handleNom(state, reader,
1605                                                         next, null);
1606                                         for (TaxonNameBase<?, ?> name : homotypicalGroup
1607                                                         .getTypifiedNames()) {
1608                                                 name.addNameTypeDesignation(speciesName, null, null,
1609                                                                 null, status, false, false, false, false);
1610                                         }
1611                                         hasNom = true;
1612                                 } else if (isStartingElement(next, ACCEPTED_NAME)) {
1613                                         handleNotYetImplementedElement(next);
1614                                 } else {
1615                                         handleUnexpectedStartElement(next);
1616                                 }
1617                         } else {
1618                                 handleUnexpectedElement(next);
1619                         }
1620                 }
1621                 // TODO handle missing end element
1622                 throw new IllegalStateException("Homotypes has no closing tag");
1623
1624         }
1625
1626         private void handleSpecimenType(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent,
1627                                 HomotypicalGroup homotypicalGroup) throws XMLStreamException {
1628                 // attributes
1629                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1630                 String typeStatus = getAndRemoveAttributeValue(attributes, TYPE_STATUS);
1631                 String notSeen = getAndRemoveAttributeValue(attributes, NOT_SEEN);
1632                 String unknown = getAndRemoveAttributeValue(attributes, UNKNOWN);
1633                 String notFound = getAndRemoveAttributeValue(attributes, NOT_FOUND);
1634                 String destroyed = getAndRemoveAttributeValue(attributes, DESTROYED);
1635                 String lost = getAndRemoveAttributeValue(attributes, LOST);
1636                 checkNoAttributes(attributes, parentEvent);
1637                 if (StringUtils.isNotEmpty(typeStatus)) {
1638                         // TODO
1639                         // currently not needed
1640                 } else if (StringUtils.isNotEmpty(notSeen)) {
1641                         handleNotYetImplementedAttribute(attributes, NOT_SEEN);
1642                 } else if (StringUtils.isNotEmpty(unknown)) {
1643                         handleNotYetImplementedAttribute(attributes, UNKNOWN);
1644                 } else if (StringUtils.isNotEmpty(notFound)) {
1645                         handleNotYetImplementedAttribute(attributes, NOT_FOUND);
1646                 } else if (StringUtils.isNotEmpty(destroyed)) {
1647                         handleNotYetImplementedAttribute(attributes, DESTROYED);
1648                 } else if (StringUtils.isNotEmpty(lost)) {
1649                         handleNotYetImplementedAttribute(attributes, LOST);
1650                 }
1651
1652                 NonViralName<?> firstName = null;
1653                 Set<TaxonNameBase> names = homotypicalGroup.getTypifiedNames();
1654                 if (names.isEmpty()) {
1655                         String message = "There is no name in a homotypical group. Can't create the specimen type";
1656                         fireWarningEvent(message, parentEvent, 8);
1657                 } else {
1658                         firstName = CdmBase.deproxy(names.iterator().next(),NonViralName.class);
1659                 }
1660
1661                 DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(DerivedUnitType.Specimen);
1662                 String text = "";
1663                 // elements
1664                 while (reader.hasNext()) {
1665                         XMLEvent next = readNoWhitespace(reader);
1666                         if (next.isEndElement()) {
1667                                 if (isMyEndingElement(next, parentEvent)) {
1668                                         makeSpecimenType(state, facade, text, firstName, parentEvent);
1669                                         return;
1670                                 } else {
1671                                         if (isEndingElement(next, FULL_TYPE)) {
1672                                                 // NOT YET IMPLEMENTED
1673                                                 popUnimplemented(next.asEndElement());
1674                                         } else if (isEndingElement(next, TYPE_STATUS)) {
1675                                                 // NOT YET IMPLEMENTED
1676                                                 popUnimplemented(next.asEndElement());
1677                                         } else if (isEndingElement(next, ORIGINAL_DETERMINATION)) {
1678                                                 // NOT YET IMPLEMENTED
1679                                                 popUnimplemented(next.asEndElement());
1680                                         } else if (isEndingElement(next, SPECIMEN_TYPE)) {
1681                                                 // NOT YET IMPLEMENTED
1682                                                 popUnimplemented(next.asEndElement());
1683                                         } else if (isEndingElement(next, COLLECTION_AND_TYPE)) {
1684                                                 // NOT YET IMPLEMENTED
1685                                                 popUnimplemented(next.asEndElement());
1686                                         } else if (isEndingElement(next, CITATION)) {
1687                                                 // NOT YET IMPLEMENTED
1688                                                 popUnimplemented(next.asEndElement());
1689                                         } else if (isEndingElement(next, NOTES)) {
1690                                                 // NOT YET IMPLEMENTED
1691                                                 popUnimplemented(next.asEndElement());
1692                                         } else if (isEndingElement(next, ANNOTATION)) {
1693                                                 // NOT YET IMPLEMENTED
1694                                                 popUnimplemented(next.asEndElement());
1695                                         } else {
1696                                                 handleUnexpectedEndElement(next.asEndElement());
1697                                         }
1698                                 }
1699                         } else if (next.isStartElement()) {
1700                                 if (isStartingElement(next, FULL_TYPE)) {
1701                                         handleNotYetImplementedElement(next);
1702                                         // homotypicalGroup = handleNom(state, reader, next, taxon,
1703                                         // homotypicalGroup);
1704                                 } else if (isStartingElement(next, TYPE_STATUS)) {
1705                                         handleNotYetImplementedElement(next);
1706                                 } else if (isStartingElement(next, GATHERING)) {
1707                                         handleGathering(state, reader, next, facade);
1708                                 } else if (isStartingElement(next, ORIGINAL_DETERMINATION)) {
1709                                         handleNotYetImplementedElement(next);
1710                                 } else if (isStartingElement(next, SPECIMEN_TYPE)) {
1711                                         handleNotYetImplementedElement(next);
1712                                 } else if (isStartingElement(next, COLLECTION_AND_TYPE)) {
1713                                         handleNotYetImplementedElement(next);
1714                                 } else if (isStartingElement(next, CITATION)) {
1715                                         handleNotYetImplementedElement(next);
1716                                 } else if (isStartingElement(next, NOTES)) {
1717                                         handleNotYetImplementedElement(next);
1718                                 } else if (isStartingElement(next, ANNOTATION)) {
1719                                         handleNotYetImplementedElement(next);
1720                                 } else {
1721                                         handleUnexpectedStartElement(next);
1722                                 }
1723                         } else if (next.isCharacters()) {
1724                                 text += next.asCharacters().getData();
1725                         } else {
1726                                 handleUnexpectedElement(next);
1727                         }
1728                 }
1729                 // TODO handle missing end element
1730                 throw new IllegalStateException("Specimen type has no closing tag");
1731         }
1732
1733         private void makeSpecimenType(MarkupImportState state, DerivedUnitFacade facade, String text,
1734                         NonViralName name, XMLEvent parentEvent) {
1735                 text = text.trim();
1736                 // remove brackets
1737                 if (text.matches("^\\(.*\\)\\.?$")) {
1738                         text = text.replaceAll("\\.", "");
1739                         text = text.substring(1, text.length() - 1);
1740                 }
1741                 String[] split = text.split("[;,]");
1742                 for (String str : split) {
1743                         str = str.trim();
1744                         boolean addToAllNamesInGroup = true;
1745                         TypeInfo typeInfo = makeSpecimenTypeTypeInfo(str, parentEvent);
1746                         SpecimenTypeDesignationStatus typeStatus = typeInfo.status;
1747                         Collection collection = createCollection(typeInfo.collectionString);
1748
1749                         // TODO improve cache strategy handling
1750                         DerivedUnitBase typeSpecimen = facade.addDuplicate(collection,
1751                                         null, null, null, null);
1752                         typeSpecimen.setCacheStrategy(new DerivedUnitFacadeCacheStrategy());
1753                         name.addSpecimenTypeDesignation((Specimen) typeSpecimen, typeStatus, null, null, null, false, addToAllNamesInGroup);
1754                 }
1755         }
1756
1757         private Collection createCollection(String code) {
1758                 // TODO deduplicate
1759                 // TODO code <-> name
1760                 Collection result = Collection.NewInstance();
1761                 result.setCode(code);
1762                 return result;
1763         }
1764
1765         private TypeInfo makeSpecimenTypeTypeInfo(String originalString, XMLEvent event) {
1766                 TypeInfo result = new TypeInfo();
1767                 String[] split = originalString.split("\\s+");
1768                 for (String str : split) {
1769                         if (str.matches(SpecimenTypeParser.typeTypePattern)) {
1770                                 SpecimenTypeDesignationStatus status;
1771                                 try {
1772                                         status = SpecimenTypeParser.parseSpecimenTypeStatus(str);
1773                                 } catch (UnknownCdmTypeException e) {
1774                                         String message = "Specimen type status '%s' not recognized by parser";
1775                                         message = String.format(message, str);
1776                                         fireWarningEvent(message, event, 4);
1777                                         status = null;
1778                                 }
1779                                 result.status = status;
1780                         } else if (str.matches(SpecimenTypeParser.collectionPattern)) {
1781                                 result.collectionString = str;
1782                         } else {
1783                                 String message = "Type part '%s' could not be recognized";
1784                                 message = String.format(message, str);
1785                                 fireWarningEvent(message, event, 2);
1786                         }
1787                 }
1788
1789                 return result;
1790         }
1791
1792
1793         private void handleGathering(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent , DerivedUnitFacade facade) throws XMLStreamException {
1794                 checkNoAttributes(parentEvent);
1795                 boolean hasCollector = false;
1796                 boolean hasFieldNum = false;
1797
1798                 // elements
1799                 while (reader.hasNext()) {
1800                         XMLEvent next = readNoWhitespace(reader);
1801                         if (next.isEndElement()) {
1802                                 if (isMyEndingElement(next, parentEvent)) {
1803                                         checkMandatoryElement(hasCollector,parentEvent.asStartElement(), COLLECTOR);
1804                                         checkMandatoryElement(hasFieldNum,parentEvent.asStartElement(), FIELD_NUM);
1805                                         return;
1806                                 } else {
1807                                         if (isEndingElement(next, ALTERNATIVE_COLLECTOR)) {
1808                                                 // NOT YET IMPLEMENTED
1809                                                 popUnimplemented(next.asEndElement());
1810                                         } else if (isEndingElement(next, ALTERNATIVE_FIELD_NUM)) {
1811                                                 // NOT YET IMPLEMENTED
1812                                                 popUnimplemented(next.asEndElement());
1813                                         } else if (isEndingElement(next, COLLECTION_TYPE_STATUS)) {
1814                                                 // NOT YET IMPLEMENTED
1815                                                 popUnimplemented(next.asEndElement());
1816                                         } else if (isEndingElement(next, COLLECTION_AND_TYPE)) {
1817                                                 // NOT YET IMPLEMENTED , does this make sense here?
1818                                                 popUnimplemented(next.asEndElement());
1819                                         } else if (isEndingElement(next,
1820                                                         ALTERNATIVE_COLLECTION_TYPE_STATUS)) {
1821                                                 // NOT YET IMPLEMENTED
1822                                                 popUnimplemented(next.asEndElement());
1823                                         } else if (isEndingElement(next, SUB_COLLECTION)) {
1824                                                 // NOT YET IMPLEMENTED
1825                                                 popUnimplemented(next.asEndElement());
1826                                         } else if (isEndingElement(next, COLLECTION)) {
1827                                                 // NOT YET IMPLEMENTED
1828                                                 popUnimplemented(next.asEndElement());
1829                                         } else if (isEndingElement(next, DATES)) {
1830                                                 // NOT YET IMPLEMENTED
1831                                                 popUnimplemented(next.asEndElement());
1832                                         } else if (isEndingElement(next, NOTES)) {
1833                                                 // NOT YET IMPLEMENTED
1834                                                 popUnimplemented(next.asEndElement());
1835                                         } else {
1836                                                 handleUnexpectedEndElement(next.asEndElement());
1837                                         }
1838                                 }
1839                         } else if (next.isStartElement()) {
1840                                 if (isStartingElement(next, COLLECTOR)) {
1841                                         hasCollector = true;
1842                                         String collectorStr = getCData(state, reader, next);
1843                                         AgentBase<?> collector = createCollector(collectorStr);
1844                                         facade.setCollector(collector);
1845                                 } else if (isStartingElement(next, ALTERNATIVE_COLLECTOR)) {
1846                                         handleNotYetImplementedElement(next);
1847                                 } else if (isStartingElement(next, FIELD_NUM)) {
1848                                         hasFieldNum = true;
1849                                         String fieldNumStr = getCData(state, reader, next);
1850                                         facade.setFieldNumber(fieldNumStr);
1851                                 } else if (isStartingElement(next, ALTERNATIVE_FIELD_NUM)) {
1852                                         handleNotYetImplementedElement(next);
1853                                 } else if (isStartingElement(next, COLLECTION_TYPE_STATUS)) {
1854                                         handleNotYetImplementedElement(next);
1855                                 } else if (isStartingElement(next, COLLECTION_AND_TYPE)) {  //does this make sense here?
1856                                         handleNotYetImplementedElement(next);
1857                                 } else if (isStartingElement(next, ALTERNATIVE_COLLECTION_TYPE_STATUS)) {
1858                                         handleNotYetImplementedElement(next);
1859                                 } else if (isStartingElement(next, SUB_COLLECTION)) {
1860                                         handleNotYetImplementedElement(next);
1861                                 } else if (isStartingElement(next, COLLECTION)) {
1862                                         handleNotYetImplementedElement(next);
1863                                 } else if (isStartingElement(next, LOCALITY)) {
1864                                         handleLocality(state, reader, next, facade);
1865                                 } else if (isStartingElement(next, DATES)) {
1866                                         handleNotYetImplementedElement(next);
1867                                 } else if (isStartingElement(next, NOTES)) {
1868                                         handleNotYetImplementedElement(next);
1869                                 } else {
1870                                         handleUnexpectedStartElement(next);
1871                                 }
1872                         } else {
1873                                 handleUnexpectedElement(next);
1874                         }
1875                 }
1876                 // TODO handle missing end element
1877                 throw new IllegalStateException("Collection has no closing tag");
1878
1879         }
1880
1881         private void handleLocality(MarkupImportState state, XMLEventReader reader,XMLEvent parentEvent, DerivedUnitFacade facade)throws XMLStreamException {
1882                 String classValue = getClassOnlyAttribute(parentEvent);
1883                 boolean isLocality = false;
1884                 NamedAreaLevel areaLevel = null;
1885                 if ("locality".equalsIgnoreCase(classValue)) {
1886                         isLocality = true;
1887                 } else {
1888                         areaLevel = makeNamedAreaLevel(state, classValue, parentEvent);
1889                 }
1890
1891                 String text = "";
1892                 // elements
1893                 while (reader.hasNext()) {
1894                         XMLEvent next = readNoWhitespace(reader);
1895                         if (next.isEndElement()) {
1896                                 if (isMyEndingElement(next, parentEvent)) {
1897                                         if (StringUtils.isNotBlank(text)) {
1898                                                 text = normalize(text);
1899                                                 if (isLocality) {
1900                                                         facade.setLocality(text);
1901                                                 } else {
1902                                                         text = CdmUtils.removeTrailingDot(text);
1903                                                         NamedArea area = makeArea(state, text, areaLevel);
1904                                                         facade.addCollectingArea(area);
1905                                                 }
1906                                         }
1907                                         // TODO
1908                                         return;
1909                                 } else {
1910                                         if (isEndingElement(next, ALTITUDE)) {
1911                                                 // NOT YET IMPLEMENTED
1912                                                 popUnimplemented(next.asEndElement());
1913                                         } else if (isEndingElement(next, COORDINATES)) {
1914                                                 // NOT YET IMPLEMENTED
1915                                                 popUnimplemented(next.asEndElement());
1916                                         } else if (isEndingElement(next, ANNOTATION)) {
1917                                                 // NOT YET IMPLEMENTED
1918                                                 popUnimplemented(next.asEndElement());
1919                                         } else {
1920                                                 handleUnexpectedEndElement(next.asEndElement());
1921                                         }
1922                                 }
1923                         } else if (next.isStartElement()) {
1924                                 if (isStartingElement(next, ALTITUDE)) {
1925                                         handleNotYetImplementedElement(next);
1926                                         // homotypicalGroup = handleNom(state, reader, next, taxon,
1927                                         // homotypicalGroup);
1928                                 } else if (isStartingElement(next, COORDINATES)) {
1929                                         handleNotYetImplementedElement(next);
1930                                 } else if (isStartingElement(next, ANNOTATION)) {
1931                                         handleNotYetImplementedElement(next);
1932                                 } else {
1933                                         handleUnexpectedStartElement(next);
1934                                 }
1935                         } else if (next.isCharacters()) {
1936                                 text += next.asCharacters().getData();
1937                         } else {
1938                                 handleUnexpectedElement(next);
1939                         }
1940                 }
1941                 throw new IllegalStateException("<SpecimenType> has no closing tag");
1942         }
1943
1944 //      private NamedArea createArea(String text, NamedAreaLevel areaLevel, MarkupImportState state) {
1945 //              NamedArea area = NamedArea.NewInstance(text, text, null);
1946 //              area.setLevel(areaLevel);
1947 //              save(area, state);
1948 //              return area;
1949 //      }
1950
1951         private AgentBase<?> createCollector(String collectorStr) {
1952                 return createAuthor(collectorStr);
1953         }
1954
1955         private String getCData(MarkupImportState state, XMLEventReader reader, XMLEvent next) throws XMLStreamException {
1956                 return getCData(state, reader, next, true);
1957         }
1958
1959         /**
1960          * Reads character data. Any element other than character data or the ending
1961          * tag will fire an unexpected element event.
1962          *
1963          * @param state
1964          * @param reader
1965          * @param next
1966          * @return
1967          * @throws XMLStreamException
1968          */
1969         private String getCData(MarkupImportState state, XMLEventReader reader, XMLEvent next,boolean checkAttributes) throws XMLStreamException {
1970                 if (checkAttributes){
1971                         checkNoAttributes(next);
1972                 }
1973
1974                 String text = "";
1975                 while (reader.hasNext()) {
1976                         XMLEvent myNext = readNoWhitespace(reader);
1977                         if (isMyEndingElement(myNext, next)) {
1978                                 return text;
1979                         } else if (myNext.isCharacters()) {
1980                                 text += myNext.asCharacters().getData();
1981                         } else {
1982                                 handleUnexpectedElement(myNext);
1983                         }
1984                 }
1985                 throw new IllegalStateException("Event has no closing tag");
1986
1987         }
1988
1989         /**
1990          * Creates the name defined by a nom tag. Adds it to the given homotypical
1991          * group (if not null).
1992          *
1993          * @param state
1994          * @param reader
1995          * @param parentEvent
1996          * @param homotypicalGroup
1997          * @return
1998          * @throws XMLStreamException
1999          */
2000         private NonViralName<?> handleNom(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent,
2001                         HomotypicalGroup homotypicalGroup) throws XMLStreamException {
2002                 boolean isSynonym = false;
2003                 boolean isNameType = state.isNameType();
2004                 // attributes
2005                 String classValue = getClassOnlyAttribute(parentEvent);
2006                 NonViralName<?> name;
2007                 if (!isNameType && ACCEPTED.equalsIgnoreCase(classValue)) {
2008                         isSynonym = false;
2009                         name = createName(state, homotypicalGroup, isSynonym);
2010                 } else if (!isNameType && SYNONYM.equalsIgnoreCase(classValue)) {
2011                         isSynonym = true;
2012                         name = createName(state, homotypicalGroup, isSynonym);
2013                 } else if (isNameType && NAME_TYPE.equalsIgnoreCase(classValue)) {
2014                         // TODO do we need to define the rank here?
2015                         name = createNameByCode(state, null);
2016                 } else {
2017                         fireUnexpectedAttributeValue(parentEvent, CLASS, classValue);
2018                         name = createNameByCode(state, null);
2019                 }
2020
2021                 Map<String, String> nameMap = new HashMap<String, String>();
2022
2023                 while (reader.hasNext()) {
2024                         XMLEvent next = readNoWhitespace(reader);
2025                         if (next.isEndElement()) {
2026                                 if (isMyEndingElement(next, parentEvent)) {
2027                                         // fill the name with all data gathered
2028                                         fillName(state, nameMap, name, next);
2029                                         return name;
2030                                 } else {
2031                                         if (isEndingElement(next, FULL_NAME)) {
2032                                                 // NOT YET IMPLEMENTED
2033                                                 popUnimplemented(next.asEndElement());
2034                                         } else if (isEndingElement(next, HOMONYM)) {
2035                                                 // NOT YET IMPLEMENTED
2036                                                 popUnimplemented(next.asEndElement());
2037                                         } else if (isEndingElement(next, NOTES)) {
2038                                                 // NOT YET IMPLEMENTED
2039                                                 popUnimplemented(next.asEndElement());
2040                                         } else if (isEndingElement(next, ANNOTATION)) {
2041                                                 // NOT YET IMPLEMENTED
2042                                                 popUnimplemented(next.asEndElement());
2043                                         } else {
2044                                                 handleUnexpectedEndElement(next.asEndElement());
2045                                         }
2046                                 }
2047                         } else if (next.isStartElement()) {
2048                                 if (isStartingElement(next, FULL_NAME)) {
2049                                         handleNotYetImplementedElement(next);
2050                                         // homotypicalGroup = handleNom(state, reader, next, taxon,
2051                                         // homotypicalGroup);
2052                                 } else if (isStartingElement(next, NUM)) {
2053                                         String num = getCData(state, reader, next);
2054                                         num = num.replace(".", "");
2055                                         num = num.replace(")", "");
2056                                         if (StringUtils.isNotBlank(num)){
2057                                                 if (state.getCurrentTaxonNum() != null &&  ! state.getCurrentTaxonNum().equals(num) ){
2058                                                         String message = "Taxontitle num and homotypes/nom/num differ ( %s <-> %s ). I use the later one.";
2059                                                         message = String.format(message, state.getCurrentTaxonNum(), num);
2060                                                         fireWarningEvent(message, next, 4);
2061                                                 }
2062                                                 state.setCurrentTaxonNum(num);
2063                                         }
2064                                 } else if (isStartingElement(next, NAME)) {
2065                                         handleName(state, reader, next, nameMap);
2066                                 } else if (isStartingElement(next, CITATION)) {
2067                                         handleCitation(state, reader, next, name);
2068                                 } else if (isStartingElement(next, HOMONYM)) {
2069                                         handleNotYetImplementedElement(next);
2070                                 } else if (isStartingElement(next, NOTES)) {
2071                                         handleNotYetImplementedElement(next);
2072                                 } else if (isStartingElement(next, ANNOTATION)) {
2073                                         handleNotYetImplementedElement(next);
2074                                 } else {
2075                                         handleUnexpectedStartElement(next);
2076                                 }
2077                         } else {
2078                                 handleUnexpectedElement(next);
2079                         }
2080                 }
2081                 // TODO handle missing end element
2082                 throw new IllegalStateException("Nom has no closing tag");
2083
2084         }
2085
2086         private void fillName(MarkupImportState state, Map<String, String> nameMap,
2087                         NonViralName name, XMLEvent event) {
2088
2089                 // Ranks: family, subfamily, tribus, genus, subgenus, section,
2090                 // subsection, species, subspecies, variety, subvariety, forma
2091                 // infrank, paraut, author, infrparaut, infraut, status, notes
2092
2093                 String infrank = getAndRemoveMapKey(nameMap, INFRANK);
2094                 String authorStr = getAndRemoveMapKey(nameMap, AUTHOR);
2095                 String paraut = getAndRemoveMapKey(nameMap, PARAUT);
2096
2097                 String infrParAut = getAndRemoveMapKey(nameMap, INFRPARAUT);
2098                 String infrAut = getAndRemoveMapKey(nameMap, INFRAUT);
2099
2100                 String statusStr = getAndRemoveMapKey(nameMap, STATUS);
2101                 String notes = getAndRemoveMapKey(nameMap, NOTES);
2102
2103                 makeRankDecision(state, nameMap, name, event, infrank);
2104
2105                 // test consistency of rank and authors
2106                 testRankAuthorConsistency(name, event, authorStr, paraut, infrParAut,infrAut);
2107
2108                 // authors
2109                 makeNomenclaturalAuthors(name, event, authorStr, paraut, infrParAut,infrAut);
2110
2111                 // status
2112                 // TODO handle pro parte, pro syn. etc.
2113                 if (StringUtils.isNotBlank(statusStr)) {
2114                         String proPartePattern = "(pro parte|p.p.)";
2115                         if (statusStr.matches(proPartePattern)) {
2116                                 state.setProParte(true);
2117                         }
2118                         try {
2119                                 // TODO handle trim earlier
2120                                 statusStr = statusStr.trim();
2121                                 NomenclaturalStatusType nomStatusType = NomenclaturalStatusType.getNomenclaturalStatusTypeByAbbreviation(statusStr);
2122                                 name.addStatus(NomenclaturalStatus.NewInstance(nomStatusType));
2123                         } catch (UnknownCdmTypeException e) {
2124                                 String message = "Status '%s' could not be recognized";
2125                                 message = String.format(message, statusStr);
2126                                 fireWarningEvent(message, event, 4);
2127                         }
2128                 }
2129
2130                 // notes
2131                 if (StringUtils.isNotBlank(notes)) {
2132                         handleNotYetImplementedAttributeValue(event, CLASS, NOTES);
2133                 }
2134
2135                 return;
2136         }
2137
2138         /**
2139          * @param state
2140          * @param nameMap
2141          * @param name
2142          * @param event
2143          * @param infrankStr
2144          */
2145         private void makeRankDecision(MarkupImportState state,
2146                         Map<String, String> nameMap, NonViralName<?> name, XMLEvent event,
2147                         String infrankStr) {
2148                 // TODO ranks
2149                 for (String key : nameMap.keySet()) {
2150                         Rank rank = makeRank(state, key, false);
2151                         if (rank == null) {
2152                                 handleNotYetImplementedAttributeValue(event, CLASS, key);
2153                         } else {
2154                                 if (name.getRank() == null || rank.isLower(name.getRank())) {
2155                                         name.setRank(rank);
2156                                 }
2157                                 String value = nameMap.get(key);
2158                                 if (rank.isSupraGeneric() || rank.isGenus()) {
2159                                         name.setGenusOrUninomial(toFirstCapital(value));
2160                                 } else if (rank.isInfraGeneric()) {
2161                                         name.setInfraGenericEpithet(toFirstCapital(value));
2162                                 } else if (rank.isSpecies()) {
2163                                         name.setSpecificEpithet(value.toLowerCase());
2164                                 } else if (rank.isInfraSpecific()) {
2165                                         name.setInfraSpecificEpithet(value.toLowerCase());
2166                                 } else {
2167                                         String message = "Invalid rank '%s'. Can't decide which epithet to fill with '%s'";
2168                                         message = String.format(message, rank.getTitleCache(),value);
2169                                         fireWarningEvent(message, event, 4);
2170                                 }
2171                         }
2172
2173                 }
2174                 // handle given infrank marker
2175                 if (StringUtils.isNotBlank(infrankStr)) {
2176                         Rank infRank = makeRank(state, infrankStr, true);
2177
2178                         if (infRank == null) {
2179                                 String message = "Infrank '%s' rank not recognized";
2180                                 message = String.format(message, infrankStr);
2181                                 fireWarningEvent(message, event, 4);
2182                         } else {
2183                                 if (name.getRank() == null) {
2184                                         name.setRank(infRank);
2185                                 } else if (infRank.isLower(name.getRank())) {
2186                                         String message = "InfRank '%s' is lower than existing rank ";
2187                                         message = String.format(message, infrankStr);
2188                                         fireWarningEvent(message, event, 2);
2189                                         name.setRank(infRank);
2190                                 } else if (infRank.equals(name.getRank())) {
2191                                         // nothing
2192                                 } else {
2193                                         String message = "InfRank '%s' is higher than existing rank ";
2194                                         message = String.format(message, infrankStr);
2195                                         fireWarningEvent(message, event, 2);
2196                                 }
2197                         }
2198                 }
2199         }
2200
2201         private String toFirstCapital(String value) {
2202                 if (StringUtils.isBlank(value)){
2203                         return value;
2204                 }else{
2205                         String result = "";
2206                         result += value.substring(0,1).toUpperCase();
2207                         if (value.length()>1){
2208                                 result += value.substring(1).toLowerCase();
2209                         }
2210                         return result;
2211                 }
2212         }
2213
2214         /**
2215          * @param name
2216          * @param event
2217          * @param authorStr
2218          * @param paraut
2219          * @param infrParAut
2220          * @param infrAut
2221          */
2222         private void makeNomenclaturalAuthors(NonViralName name, XMLEvent event,
2223                                 String authorStr, String paraut, String infrParAut, String infrAut) {
2224                 if (name.getRank() != null && name.getRank().isInfraSpecific()) {
2225                         if (StringUtils.isNotBlank(infrAut)) {
2226                                 INomenclaturalAuthor[] authorAndEx = authorAndEx(infrAut, event);
2227                                 name.setCombinationAuthorTeam(authorAndEx[0]);
2228                                 name.setExCombinationAuthorTeam(authorAndEx[1]);
2229                         }
2230                         if (StringUtils.isNotBlank(infrParAut)) {
2231                                 INomenclaturalAuthor[] authorAndEx = authorAndEx(infrParAut, event);
2232                                 name.setBasionymAuthorTeam(authorAndEx[0]);
2233                                 name.setExBasionymAuthorTeam(authorAndEx[1]);
2234                         }
2235                 } else {
2236                         if (name.getRank() == null){
2237                                 String message = "No rank defined. Check correct usage of authors!";
2238                                 fireWarningEvent(message, event, 4);
2239                                 if (isNotBlank(infrParAut) || isNotBlank(infrAut)){
2240                                         authorStr = infrAut;
2241                                         paraut = infrParAut;
2242                                 }
2243                         }
2244                         if (StringUtils.isNotBlank(authorStr)) {
2245                                 INomenclaturalAuthor[] authorAndEx = authorAndEx(authorStr, event);
2246                                 name.setCombinationAuthorTeam(authorAndEx[0]);
2247                                 name.setExCombinationAuthorTeam(authorAndEx[1]);
2248                         }
2249                         if (StringUtils.isNotBlank(paraut)) {
2250                                 INomenclaturalAuthor[] authorAndEx = authorAndEx(paraut, event);
2251                                 name.setBasionymAuthorTeam(authorAndEx[0]);
2252                                 name.setExBasionymAuthorTeam(authorAndEx[1]);
2253                         }
2254                 }
2255         }
2256
2257         private TeamOrPersonBase[] authorAndEx(String authorAndEx, XMLEvent xmlEvent) {
2258                 authorAndEx = authorAndEx.trim();
2259                 TeamOrPersonBase[] result = new TeamOrPersonBase[2];
2260
2261                 String[] split = authorAndEx.split("\\sex\\s");
2262                 if (split.length > 2) {
2263                         String message = "There is more then 1 ' ex ' in author string. Can't separate author and ex-author";
2264                         fireWarningEvent(message, xmlEvent, 4);
2265                         result[0] = createAuthor(authorAndEx);
2266                 } else if (split.length == 2) {
2267                         result[0] = createAuthor(split[1]);
2268                         result[1] = createAuthor(split[0]);
2269                 } else {
2270                         result[0] = createAuthor(split[0]);
2271                 }
2272                 return result;
2273         }
2274
2275         /**
2276          * Tests if the names rank is consistent with the given author strings.
2277          * @param name
2278          * @param event
2279          * @param authorStr
2280          * @param paraut
2281          * @param infrParAut
2282          * @param infrAut
2283          */
2284         private void testRankAuthorConsistency(NonViralName name, XMLEvent event,
2285                                 String authorStr, String paraut, String infrParAut, String infrAut) {
2286                 if (name.getRank() == null){
2287                         return;
2288                 }
2289                 if (name.getRank().isInfraSpecific()) {
2290                         if (StringUtils.isBlank(infrParAut)
2291                                         && StringUtils.isBlank(infrAut)    //was isNotBlank before 29.5.2012
2292                                         && (StringUtils.isNotBlank(paraut) || StringUtils.isNotBlank(authorStr))
2293                                         && ! name.isAutonym()) {
2294                                 String message = "Rank is infraspecicific but has only specific or higher author(s)";
2295                                 fireWarningEvent(message, event, 4);
2296                         }
2297                 } else {
2298                         // is not infraspecific
2299                         if (StringUtils.isNotBlank(infrParAut)  || StringUtils.isNotBlank(infrAut)) {
2300                                 String message = "Rank is not infraspecicific but name has infra author(s)";
2301                                 fireWarningEvent(message, event, 4);
2302                         }
2303                 }
2304         }
2305
2306         /**
2307          * Returns the (empty) name with the correct homotypical group depending on
2308          * the taxon status. Throws NPE if no currentTaxon is set in state.
2309          *
2310          * @param state
2311          * @param homotypicalGroup
2312          * @param isSynonym
2313          * @return
2314          */
2315         private NonViralName<?> createName(MarkupImportState state,
2316                         HomotypicalGroup homotypicalGroup, boolean isSynonym) {
2317                 NonViralName<?> name;
2318                 Taxon taxon = state.getCurrentTaxon();
2319                 if (isSynonym) {
2320                         Rank defaultRank = Rank.SPECIES(); // can be any
2321                         name = createNameByCode(state, defaultRank);
2322                         if (homotypicalGroup != null) {
2323                                 name.setHomotypicalGroup(homotypicalGroup);
2324                         }
2325                         SynonymRelationshipType synonymType = SynonymRelationshipType.HETEROTYPIC_SYNONYM_OF();
2326                         if (taxon.getHomotypicGroup().equals(homotypicalGroup)) {
2327                                 synonymType = SynonymRelationshipType.HOMOTYPIC_SYNONYM_OF();
2328                         }
2329                         taxon.addSynonymName(name, synonymType);
2330                 } else {
2331                         name = CdmBase.deproxy(taxon.getName(), NonViralName.class);
2332                 }
2333                 return name;
2334         }
2335
2336         private void handleName(MarkupImportState state, XMLEventReader reader,
2337                         XMLEvent parentEvent, Map<String, String> nameMap)
2338                         throws XMLStreamException {
2339                 String classValue = getClassOnlyAttribute(parentEvent);
2340
2341                 String text = "";
2342                 while (reader.hasNext()) {
2343                         XMLEvent next = readNoWhitespace(reader);
2344                         if (isMyEndingElement(next, parentEvent)) {
2345                                 nameMap.put(classValue, text);
2346                                 return;
2347                         } else if (next.isStartElement()) {
2348                                 if (isStartingElement(next, ANNOTATION)) {
2349                                         handleNotYetImplementedElement(next);
2350                                 } else {
2351                                         handleUnexpectedStartElement(next.asStartElement());
2352                                 }
2353                         } else if (next.isCharacters()) {
2354                                 text += next.asCharacters().getData();
2355                         } else {
2356                                 handleUnexpectedEndElement(next.asEndElement());
2357                         }
2358                 }
2359                 throw new IllegalStateException("name has no closing tag");
2360
2361         }
2362
2363         /**
2364          * @param state
2365          * @param classValue
2366          * @param byAbbrev
2367          * @return
2368          */
2369         private Rank makeRank(MarkupImportState state, String value,
2370                         boolean byAbbrev) {
2371                 Rank rank = null;
2372                 if (StringUtils.isBlank(value)) {
2373                         return null;
2374                 }
2375                 try {
2376                         boolean useUnknown = true;
2377                         NomenclaturalCode nc = makeNomenclaturalCode(state);
2378                         if (byAbbrev) {
2379                                 rank = Rank.getRankByAbbreviation(value, nc, useUnknown);
2380                         } else {
2381                                 rank = Rank.getRankByEnglishName(value, nc, useUnknown);
2382                         }
2383                         if (rank.equals(Rank.UNKNOWN_RANK())) {
2384                                 rank = null;
2385                         }
2386                 } catch (UnknownCdmTypeException e) {
2387                         // doNothing
2388                 }
2389                 return rank;
2390         }
2391
2392         // public void handleNameNotRank(MarkupImportState state, XMLEventReader
2393         // reader, XMLEvent parentEvent, String classValue, NonViralName name)
2394         // throws XMLStreamException {
2395         // if (ACCEPTED.equalsIgnoreCase(classValue)){
2396         // }else if (SYNONYM.equalsIgnoreCase(classValue)){
2397         // }else{
2398         // //TODO Not yet implemented
2399         // handleNotYetImplementedAttributeValue(parentEvent, CLASS, classValue);
2400         // }
2401         // }
2402
2403         private void handleCitation(MarkupImportState state, XMLEventReader reader,     XMLEvent parentEvent, NonViralName name) throws XMLStreamException {
2404                 String classValue = getClassOnlyAttribute(parentEvent);
2405
2406                 state.setCitation(true);
2407                 boolean hasRefPart = false;
2408                 Map<String, String> refMap = new HashMap<String, String>();
2409                 while (reader.hasNext()) {
2410                         XMLEvent next = readNoWhitespace(reader);
2411                         if (isMyEndingElement(next, parentEvent)) {
2412                                 checkMandatoryElement(hasRefPart, parentEvent.asStartElement(),
2413                                                 REF_PART);
2414                                 Reference<?> reference = createReference(state, refMap, next);
2415                                 String microReference = refMap.get(DETAILS);
2416                                 doCitation(state, name, classValue, reference, microReference,
2417                                                 parentEvent);
2418                                 state.setCitation(false);
2419                                 return;
2420                         } else if (isStartingElement(next, REF_PART)) {
2421                                 handleRefPart(state, reader, next, refMap);
2422                                 hasRefPart = true;
2423                         } else {
2424                                 handleUnexpectedElement(next);
2425                         }
2426                 }
2427                 throw new IllegalStateException("Citation has no closing tag");
2428
2429         }
2430
2431         private void handleRefPart(MarkupImportState state, XMLEventReader reader,XMLEvent parentEvent, Map<String, String> refMap) throws XMLStreamException {
2432                 String classValue = getClassOnlyAttribute(parentEvent);
2433
2434                 String text = "";
2435                 while (reader.hasNext()) {
2436                         XMLEvent next = readNoWhitespace(reader);
2437                         if (isMyEndingElement(next, parentEvent)) {
2438                                 refMap.put(classValue, text);
2439                                 return;
2440                         } else if (next.isStartElement()) {
2441                                 if (isStartingElement(next, ANNOTATION)) {
2442                                         handleNotYetImplementedElement(next);
2443                                 } else if (isStartingElement(next, ITALICS)) {
2444                                         handleNotYetImplementedElement(next);
2445                                 } else if (isStartingElement(next, BOLD)) {
2446                                         handleNotYetImplementedElement(next);
2447                                 } else {
2448                                         handleUnexpectedStartElement(next.asStartElement());
2449                                 }
2450                         } else if (next.isCharacters()) {
2451                                 text += next.asCharacters().getData();
2452                         } else {
2453                                 handleUnexpectedEndElement(next.asEndElement());
2454                         }
2455                 }
2456                 throw new IllegalStateException("RefPart has no closing tag");
2457
2458         }
2459
2460         private Reference<?> createReference(MarkupImportState state, Map<String, String> refMap, XMLEvent parentEvent) {
2461                 // TODO
2462                 Reference<?> reference;
2463
2464                 String type = getAndRemoveMapKey(refMap, PUBTYPE);
2465                 String authorStr = getAndRemoveMapKey(refMap, AUTHOR);
2466                 String titleStr = getAndRemoveMapKey(refMap, PUBTITLE);
2467                 String titleCache = getAndRemoveMapKey(refMap, PUBFULLNAME);
2468                 String volume = getAndRemoveMapKey(refMap, VOLUME);
2469                 String edition = getAndRemoveMapKey(refMap, EDITION);
2470                 String editors = getAndRemoveMapKey(refMap, EDITORS);
2471                 String year = getAndRemoveMapKey(refMap, YEAR);
2472                 String pubName = getAndRemoveMapKey(refMap, PUBNAME);
2473                 String pages = getAndRemoveMapKey(refMap, PAGES);
2474
2475                 if (state.isCitation()) {
2476                         if (volume != null || "journal".equalsIgnoreCase(type)) {
2477                                 IArticle article = ReferenceFactory.newArticle();
2478                                 if (pubName != null) {
2479                                         IJournal journal = ReferenceFactory.newJournal();
2480                                         journal.setTitle(pubName);
2481                                         article.setInJournal(journal);
2482                                 }
2483                                 reference = (Reference<?>) article;
2484
2485                         } else {
2486                                 // TODO
2487                                 if (pubName != null){
2488                                         reference  = ReferenceFactory.newBookSection();
2489                                 }else{
2490                                         reference = ReferenceFactory.newBook();
2491                                 }
2492                         }
2493                         // TODO use existing author from name or before
2494                         TeamOrPersonBase<?> author = createAuthor(authorStr);
2495                         reference.setAuthorTeam(author);
2496
2497                         reference.setTitle(titleStr);
2498                         if (StringUtils.isNotBlank(titleCache)) {
2499                                 reference.setTitleCache(titleCache, true);
2500                         }
2501                         reference.setEdition(edition);
2502                         reference.setEditor(editors);
2503
2504                         if (pubName != null) {
2505                                 Reference<?> inReference;
2506                                 if (reference.getType().equals(ReferenceType.Article)) {
2507                                         inReference = ReferenceFactory.newJournal();
2508                                 } else {
2509                                         inReference = ReferenceFactory.newGeneric();
2510                                 }
2511                                 inReference.setTitle(pubName);
2512                                 reference.setInReference(inReference);
2513                         }
2514
2515
2516                 } else {  //no citation
2517                         if (volume != null || "journal".equalsIgnoreCase(type)) {
2518                                 IArticle article = ReferenceFactory.newArticle();
2519                                 if (pubName != null) {
2520                                         IJournal journal = ReferenceFactory.newJournal();
2521                                         journal.setTitle(pubName);
2522                                         article.setInJournal(journal);
2523                                 }
2524                                 reference = (Reference<?>) article;
2525
2526                         } else {
2527                                 Reference<?> bookOrPartOf = ReferenceFactory.newGeneric();
2528                                 reference = bookOrPartOf;
2529                         }
2530
2531                         // TODO type
2532                         TeamOrPersonBase<?> author = createAuthor(authorStr);
2533                         reference.setAuthorTeam(author);
2534
2535                         reference.setTitle(titleStr);
2536                         if (StringUtils.isNotBlank(titleCache)) {
2537                                 reference.setTitleCache(titleCache, true);
2538                         }
2539                         reference.setEdition(edition);
2540                         reference.setEditor(editors);
2541
2542                         if (pubName != null) {
2543                                 Reference<?> inReference;
2544                                 if (reference.getType().equals(ReferenceType.Article)) {
2545                                         inReference = ReferenceFactory.newJournal();
2546                                 } else {
2547                                         inReference = ReferenceFactory.newGeneric();
2548                                 }
2549                                 inReference.setTitle(pubName);
2550                                 reference.setInReference(inReference);
2551                         }
2552                 }
2553                 reference.setVolume(volume);
2554                 reference.setDatePublished(TimePeriod.parseString(year));
2555                 //TODO check if this is handled correctly in FM markup
2556                 reference.setPages(pages);
2557
2558                 // TODO
2559                 String[] unhandledList = new String[]{ALTERNATEPUBTITLE, ISSUE, NOTES, STATUS};
2560                 for (String unhandled : unhandledList){
2561                         String value = getAndRemoveMapKey(refMap, unhandled);
2562                         if (isNotBlank(value)){
2563                                 this.handleNotYetImplementedAttributeValue(parentEvent, CLASS, unhandled);
2564                         }
2565                 }
2566
2567                 for (String key : refMap.keySet()) {
2568                         if (!DETAILS.equalsIgnoreCase(key)) {
2569                                 this.fireUnexpectedAttributeValue(parentEvent, CLASS, key);
2570                         }
2571                 }
2572
2573                 return reference;
2574         }
2575
2576         private TeamOrPersonBase createAuthor(String authorTitle) {
2577                 // TODO atomize and also use by name creation
2578                 TeamOrPersonBase result = Team.NewTitledInstance(authorTitle,
2579                                 authorTitle);
2580                 return result;
2581         }
2582
2583         private String getAndRemoveMapKey(Map<String, String> map, String key) {
2584                 String result = map.get(key);
2585                 map.remove(key);
2586                 if (result != null) {
2587                         result = normalize(result);
2588                 }
2589                 return StringUtils.stripToNull(result);
2590         }
2591
2592         private void doCitation(MarkupImportState state, NonViralName name,
2593                         String classValue, Reference reference, String microCitation,
2594                         XMLEvent parentEvent) {
2595                 if (PUBLICATION.equalsIgnoreCase(classValue)) {
2596                         name.setNomenclaturalReference(reference);
2597                         name.setNomenclaturalMicroReference(microCitation);
2598                 } else if (USAGE.equalsIgnoreCase(classValue)) {
2599                         Taxon taxon = state.getCurrentTaxon();
2600                         TaxonDescription td = getTaxonDescription(taxon, state
2601                                         .getConfig().getSourceReference(), false, true);
2602                         TextData citation = TextData.NewInstance(Feature.CITATION());
2603                         // TODO name used in source
2604                         citation.addSource(null, null, reference, microCitation);
2605                         td.addElement(citation);
2606                 } else if (TYPE.equalsIgnoreCase(classValue)) {
2607                         handleNotYetImplementedAttributeValue(parentEvent, CLASS,
2608                                         classValue);
2609                 } else {
2610                         // TODO Not yet implemented
2611                         handleNotYetImplementedAttributeValue(parentEvent, CLASS,
2612                                         classValue);
2613                 }
2614         }
2615
2616         private void handleFeature(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
2617                 String classValue = getClassOnlyAttribute(parentEvent);
2618                 Feature feature = makeFeature(classValue, state, parentEvent);
2619                 Taxon taxon = state.getCurrentTaxon();
2620                 TaxonDescription taxonDescription = getTaxonDescription(taxon, state.getConfig().getSourceReference(), NO_IMAGE_GALLERY, CREATE_NEW);
2621                 // TextData figureHolderTextData = null; //for use with one TextData for
2622                 // all figure only
2623
2624                 boolean isDescription = feature.equals(Feature.DESCRIPTION());
2625                 DescriptionElementBase lastDescriptionElement = null;
2626
2627                 while (reader.hasNext()) {
2628                         XMLEvent next = readNoWhitespace(reader);
2629                         if (isMyEndingElement(next, parentEvent)) {
2630                                 return;
2631                         } else if (isEndingElement(next, DISTRIBUTION_LIST) || isEndingElement(next, HABITAT_LIST)) {
2632                                 // only handle list elements
2633                         } else if (isStartingElement(next, HEADING)) {
2634                                 makeFeatureHeading(state, reader, classValue, feature, next);
2635                         } else if (isStartingElement(next, WRITER)) {
2636                                 makeFeatureWriter(state, reader, feature, taxon, next);
2637 //                      } else if (isStartingElement(next, DISTRIBUTION_LOCALITY)) {
2638 //                              if (!feature.equals(Feature.DISTRIBUTION())) {
2639 //                                      String message = "Distribution locality only allowed for feature of type 'distribution'";
2640 //                                      fireWarningEvent(message, next, 4);
2641 //                              }
2642 //                              handleDistributionLocality(state, reader, next);
2643                         } else if (isStartingElement(next, DISTRIBUTION_LIST) || isStartingElement(next, HABITAT_LIST)) {
2644                                 // only handle single list elements
2645                         } else if (isStartingElement(next, HABITAT)) {
2646                                 if (!(feature.equals(Feature.HABITAT())
2647                                                 || feature.equals(Feature.HABITAT_ECOLOGY())
2648                                                 || feature.equals(Feature.ECOLOGY()))) {
2649                                         String message = "Habitat only allowed for feature of type 'habitat','habitat ecology' or 'ecology'";
2650                                         fireWarningEvent(message, next, 4);
2651                                 }
2652                                 handleHabitat(state, reader, next);
2653                         } else if (isStartingElement(next, CHAR)) {
2654                                 TextData textData = handleChar(state, reader, next);
2655                                 taxonDescription.addElement(textData);
2656                         } else if (isStartingElement(next, STRING)) {
2657                                 lastDescriptionElement = makeFeatureString(state, reader,feature, taxonDescription, lastDescriptionElement,next);
2658                         } else if (isStartingElement(next, FIGURE_REF)) {
2659                                 lastDescriptionElement = makeFeatureFigureRef(state, reader, taxonDescription, isDescription, lastDescriptionElement, next);
2660                         } else if (isStartingElement(next, REFERENCES)) {
2661                                 // TODO details/microcitation ??
2662
2663                                 List<Reference<?>> refs = handleReferences(state, reader, next);
2664                                 if (!refs.isEmpty()) {
2665                                         // TODO
2666                                         Reference<?> descriptionRef = state.getConfig().getSourceReference();
2667                                         TaxonDescription description = getTaxonDescription(taxon, descriptionRef, false, true);
2668                                         TextData featurePlaceholder = docImport.getFeaturePlaceholder(state, description, feature, true);
2669                                         for (Reference<?> citation : refs) {
2670                                                 featurePlaceholder.addSource(null, null, citation, null);
2671                                         }
2672                                 } else {
2673                                         String message = "No reference found in references";
2674                                         fireWarningEvent(message, next, 6);
2675                                 }
2676                         } else if (isStartingElement(next, NUM)) {
2677                                 //TODO
2678                                 handleNotYetImplementedElement(next);
2679                         } else if (isEndingElement(next, NUM)) {
2680                                 //TODO
2681                                 popUnimplemented(next.asEndElement());
2682                         } else {
2683                                 handleUnexpectedElement(next);
2684                         }
2685                 }
2686                 throw new IllegalStateException("<Feature> has no closing tag");
2687         }
2688
2689         /**
2690          * @param state
2691          * @param reader
2692          * @param taxonDescription
2693          * @param isDescription
2694          * @param lastDescriptionElement
2695          * @param next
2696          * @return
2697          * @throws XMLStreamException
2698          */
2699         private DescriptionElementBase makeFeatureFigureRef(MarkupImportState state, XMLEventReader reader,TaxonDescription taxonDescription,
2700                                         boolean isDescription, DescriptionElementBase lastDescriptionElement, XMLEvent next)throws XMLStreamException {
2701                 FigureDataHolder figureHolder = handleFigureRef(state, reader, next);
2702                 Feature figureFeature = getFeature(state,MarkupTransformer.uuidFigures, "Figures", "Figures", "Fig.",null);
2703                 if (isDescription) {
2704                         TextData figureHolderTextData = null;
2705                         // if (figureHolderTextData == null){
2706                         figureHolderTextData = TextData.NewInstance(figureFeature);
2707                         if (StringUtils.isNotBlank(figureHolder.num)) {
2708                                 String annotationText = "<num>" + figureHolder.num.trim()
2709                                                 + "</num>";
2710                                 Annotation annotation = Annotation.NewInstance(annotationText,
2711                                                 AnnotationType.TECHNICAL(), Language.DEFAULT());
2712                                 figureHolderTextData.addAnnotation(annotation);
2713                         }
2714                         if (StringUtils.isNotBlank(figureHolder.figurePart)) {
2715                                 String annotationText = "<figurePart>"+ figureHolder.figurePart.trim() + "</figurePart>";
2716                                 Annotation annotation = Annotation.NewInstance(annotationText,AnnotationType.EDITORIAL(), Language.DEFAULT());
2717                                 figureHolderTextData.addAnnotation(annotation);
2718                         }
2719                         // if (StringUtils.isNotBlank(figureText)){
2720                         // figureHolderTextData.putText(Language.DEFAULT(), figureText);
2721                         // }
2722                         taxonDescription.addElement(figureHolderTextData);
2723                         // }
2724                         registerFigureDemand(state, figureHolderTextData, figureHolder.ref);
2725                 } else {
2726                         if (lastDescriptionElement == null) {
2727                                 String message = "No description element created yet that can be referred by figure. Create new TextData instead";
2728                                 fireWarningEvent(message, next, 4);
2729                                 lastDescriptionElement = TextData.NewInstance(figureFeature);
2730                                 taxonDescription.addElement(lastDescriptionElement);
2731                         }
2732                         registerFigureDemand(state, lastDescriptionElement,
2733                                         figureHolder.ref);
2734                 }
2735                 return lastDescriptionElement;
2736         }
2737
2738         /**
2739          * @param state
2740          * @param reader
2741          * @param feature
2742          * @param taxonDescription
2743          * @param lastDescriptionElement
2744          * @param distributionList
2745          * @param next
2746          * @return
2747          * @throws XMLStreamException
2748          */
2749         private DescriptionElementBase makeFeatureString(MarkupImportState state,XMLEventReader reader, Feature feature,
2750                                 TaxonDescription taxonDescription, DescriptionElementBase lastDescriptionElement, XMLEvent next) throws XMLStreamException {
2751                 //for specimen only
2752                 if (feature.equals(Feature.SPECIMEN()) || feature.equals(Feature.MATERIALS_EXAMINED())){
2753
2754                         List<DescriptionElementBase> specimens = handleMaterialsExamined(state, reader, next);
2755                         for (DescriptionElementBase specimen : specimens){
2756                                 taxonDescription.addElement(specimen);
2757                                 lastDescriptionElement = specimen;
2758                         }
2759                         return lastDescriptionElement;
2760                 }
2761
2762                 //others
2763                 Map<String, String> subheadingMap = handleString(state, reader, next, feature);
2764                 for (String subheading : subheadingMap.keySet()) {
2765                         Feature subheadingFeature = feature;
2766                         if (StringUtils.isNotBlank(subheading) && subheadingMap.size() > 1) {
2767                                 subheadingFeature = makeFeature(subheading, state, next);
2768                         }
2769                         if (feature.equals(Feature.COMMON_NAME())){
2770                                 List<DescriptionElementBase> commonNames = makeVernacular(state, subheading, subheadingMap.get(subheading));
2771                                 for (DescriptionElementBase commonName : commonNames){
2772                                         taxonDescription.addElement(commonName);
2773                                         lastDescriptionElement = commonName;
2774                                 }
2775                         }else {
2776                                 TextData textData = TextData.NewInstance(subheadingFeature);
2777                                 textData.putText(Language.DEFAULT(), subheadingMap.get(subheading));
2778                                 taxonDescription.addElement(textData);
2779                                 lastDescriptionElement = textData;
2780                                 // TODO how to handle figures when these data are split in
2781                                 // subheadings
2782                         }
2783                 }
2784                 return lastDescriptionElement;
2785         }
2786
2787         private List<DescriptionElementBase> makeVernacular(MarkupImportState state, String subheading, String commonNameString) throws XMLStreamException {
2788                 List<DescriptionElementBase> result = new ArrayList<DescriptionElementBase>();
2789                 String[] splits = commonNameString.split(",");
2790                 for (String split : splits){
2791                         split = split.trim();
2792                         if (! split.matches(".*\\(.*\\)\\.?")){
2793                                 fireWarningEvent("Common name string '"+split+"' does not match given pattern", state.getReader().peek(), 4);
2794                         }
2795
2796                         String name = split.replaceAll("\\(.*\\)", "").replace(".", "").trim();
2797                         String languageStr = split.replaceFirst(".*\\(", "").replaceAll("\\)\\.?", "").trim();
2798
2799                         Language language = null;
2800                         if (StringUtils.isNotBlank(languageStr)){
2801                                 try {
2802                                         UUID langUuid = state.getTransformer().getLanguageUuid(languageStr);
2803                                         TermVocabulary<?> voc = null;
2804                                         language = getLanguage(state, langUuid, languageStr, languageStr, null, voc);
2805                                         if (language == null){
2806                                                 logger.warn("Language " + languageStr + " not recognized by transformer");
2807                                         }
2808                                 } catch (UndefinedTransformerMethodException e) {
2809                                         throw new RuntimeException(e);
2810                                 }
2811                         }
2812                         NamedArea area = null;
2813                         CommonTaxonName commonTaxonName = CommonTaxonName.NewInstance(name, language, area);
2814                         result.add(commonTaxonName);
2815                 }
2816
2817                 return result;
2818         }
2819
2820         private List<DescriptionElementBase>  handleMaterialsExamined(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
2821                 List<DescriptionElementBase> result = new ArrayList<DescriptionElementBase>();
2822                 while (reader.hasNext()) {
2823                         XMLEvent next = readNoWhitespace(reader);
2824                         if (isMyEndingElement(next, parentEvent)) {
2825                                 if (result.isEmpty()){
2826                                         fireWarningEvent("Materials examined created empty Individual Associations list", parentEvent, 4);
2827                                 }
2828                                 return result;
2829                         } else if (isStartingElement(next, SUB_HEADING)) {
2830                                 handleNotYetImplementedElement(next);
2831                         } else if (isStartingElement(next, BR)) {
2832                                 handleNotYetImplementedElement(next);
2833                         } else if (isStartingElement(next, GATHERING)) {
2834                                 DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(DerivedUnitType.DerivedUnit.DerivedUnit);
2835                                 handleGathering(state, reader, next, facade);
2836                                 SpecimenOrObservationBase<?> specimen;
2837                                 if (facade.innerDerivedUnit() != null){
2838                                         specimen = facade.innerDerivedUnit();
2839                                 }else{
2840                                         specimen = facade.innerFieldObservation();
2841                                 }
2842                                 IndividualsAssociation individualsAssociation = IndividualsAssociation.NewInstance();
2843                                 individualsAssociation.setAssociatedSpecimenOrObservation(specimen);
2844                                 result.add(individualsAssociation);
2845                         } else {
2846                                 handleUnexpectedElement(next);
2847                         }
2848                 }
2849                 throw new IllegalStateException("<String> has no closing tag");
2850
2851         }
2852
2853         /**
2854          * @param state
2855          * @param reader
2856          * @param feature
2857          * @param taxon
2858          * @param next
2859          * @throws XMLStreamException
2860          */
2861         private void makeFeatureWriter(MarkupImportState state,XMLEventReader reader, Feature feature, Taxon taxon, XMLEvent next) throws XMLStreamException {
2862                 WriterDataHolder writer = handleWriter(state, reader, next);
2863                 if (isNotBlank(writer.writer)) {
2864                         // TODO
2865                         Reference<?> ref = state.getConfig().getSourceReference();
2866                         TaxonDescription description = getTaxonDescription(taxon, ref,
2867                                         false, true);
2868                         TextData featurePlaceholder = docImport.getFeaturePlaceholder(state,
2869                                         description, feature, true);
2870                         featurePlaceholder.addAnnotation(writer.annotation);
2871                         registerFootnotes(state, featurePlaceholder, writer.footnotes);
2872                 } else {
2873                         String message = "Writer element is empty";
2874                         fireWarningEvent(message, next, 4);
2875                 }
2876         }
2877
2878         /**
2879          * @param state
2880          * @param reader
2881          * @param classValue
2882          * @param feature
2883          * @param next
2884          * @throws XMLStreamException
2885          */
2886         private void makeFeatureHeading(MarkupImportState state, XMLEventReader reader, String classValue, Feature feature, XMLEvent next) throws XMLStreamException {
2887                 String heading = handleHeading(state, reader, next);
2888                 if (StringUtils.isNotBlank(heading)) {
2889                         if (!heading.equalsIgnoreCase(classValue)) {
2890                                 try {
2891                                         if (!feature.equals(state.getTransformer().getFeatureByKey(
2892                                                         heading))) {
2893                                                 UUID headerFeatureUuid = state.getTransformer()
2894                                                                 .getFeatureUuid(heading);
2895                                                 if (!feature.getUuid().equals(headerFeatureUuid)) {
2896                                                         String message = "Feature heading '%s' differs from feature class '%s' and can not be transformed to feature";
2897                                                         message = String.format(message, heading,
2898                                                                         classValue);
2899                                                         fireWarningEvent(message, next, 1);
2900                                                 }
2901                                         }
2902                                 } catch (UndefinedTransformerMethodException e) {
2903                                         throw new RuntimeException(e);
2904                                 }
2905                         } else {
2906                                 // do nothing
2907                         }
2908                 }
2909         }
2910
2911         private List<Reference<?>> handleReferences(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
2912                 // attributes
2913                 Map<String, Attribute> attributes = getAttributes(parentEvent);
2914                 String bibliography = getAndRemoveAttributeValue(attributes,
2915                                 BIBLIOGRAPHY);
2916                 String serialsAbbreviations = getAndRemoveAttributeValue(attributes,
2917                                 SERIALS_ABBREVIATIONS);
2918                 if (isNotBlank(bibliography) || isNotBlank(serialsAbbreviations)) {
2919                         String message = "Attributes not yet implemented for <references>";
2920                         fireWarningEvent(message, parentEvent, 4);
2921                 }
2922
2923                 List<Reference<?>> result = new ArrayList<Reference<?>>();
2924
2925                 // elements
2926                 while (reader.hasNext()) {
2927                         XMLEvent next = readNoWhitespace(reader);
2928                         if (next.isEndElement()) {
2929                                 if (isMyEndingElement(next, parentEvent)) {
2930                                         return result;
2931                                 } else {
2932                                         if (isEndingElement(next, HEADING)) {
2933                                                 // NOT YET IMPLEMENTED
2934                                                 popUnimplemented(next.asEndElement());
2935                                         } else if (isEndingElement(next, WRITER)) {
2936                                                 // NOT YET IMPLEMENTED
2937                                                 popUnimplemented(next.asEndElement());
2938                                         } else if (isEndingElement(next, FOOTNOTE)) {
2939                                                 // NOT YET IMPLEMENTED
2940                                                 popUnimplemented(next.asEndElement());
2941                                         } else if (isEndingElement(next, STRING)) {
2942                                                 // NOT YET IMPLEMENTED
2943                                                 popUnimplemented(next.asEndElement());
2944                                         } else if (isEndingElement(next, REF_NUM)) {
2945                                                 // NOT YET IMPLEMENTED
2946                                                 popUnimplemented(next.asEndElement());
2947                                         } else {
2948                                                 handleUnexpectedEndElement(next.asEndElement());
2949                                         }
2950                                 }
2951                         } else if (next.isStartElement()) {
2952                                 if (isStartingElement(next, HEADING)) {
2953                                         handleNotYetImplementedElement(next);
2954                                 } else if (isStartingElement(next, SUB_HEADING)) {
2955                                         String subheading = getCData(state, reader, next).trim();
2956                                         String excludePattern = "(i?)(References?|Literature):?";
2957                                         if (!subheading.matches(excludePattern)) {
2958                                                 fireNotYetImplementedElement(next.getLocation(), next.asStartElement().getName(), 0);
2959                                         }
2960                                 } else if (isStartingElement(next, WRITER)) {
2961                                         handleNotYetImplementedElement(next);
2962                                 } else if (isStartingElement(next, FOOTNOTE)) {
2963                                         handleNotYetImplementedElement(next);
2964                                 } else if (isStartingElement(next, STRING)) {
2965                                         handleNotYetImplementedElement(next);
2966                                 } else if (isStartingElement(next, REF_NUM)) {
2967                                         handleNotYetImplementedElement(next);
2968                                 } else if (isStartingElement(next, REFERENCE)) {
2969                                         Reference<?> ref = handleReference(state, reader, next);
2970                                         result.add(ref);
2971                                 } else {
2972                                         handleUnexpectedStartElement(next);
2973                                 }
2974                         } else {
2975                                 handleUnexpectedElement(next);
2976                         }
2977                 }
2978                 throw new IllegalStateException("<References> has no closing tag");
2979         }
2980
2981         private void handleHabitat(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
2982                 checkNoAttributes(parentEvent);
2983                 Taxon taxon = state.getCurrentTaxon();
2984                 // TODO which ref to take?
2985                 Reference<?> ref = state.getConfig().getSourceReference();
2986
2987                 String text = "";
2988                 while (reader.hasNext()) {
2989                         XMLEvent next = readNoWhitespace(reader);
2990                         if (isMyEndingElement(next, parentEvent)) {
2991                                 TaxonDescription description = getTaxonDescription(taxon, ref,
2992                                                 false, true);
2993                                 UUID uuidExtractedHabitat = MarkupTransformer.uuidExtractedHabitat;
2994                                 Feature feature = getFeature(
2995                                                 state,
2996                                                 uuidExtractedHabitat,
2997                                                 "Extracted Habitat",
2998                                                 "An structured habitat that was extracted from a habitat text",
2999                                                 "extr. habit.", null);
3000                                 TextData habitat = TextData.NewInstance(feature);
3001                                 habitat.putText(Language.DEFAULT(), text);
3002                                 description.addElement(habitat);
3003
3004                                 return;
3005                         } else if (next.isStartElement()) {
3006                                 if (isStartingElement(next, ALTITUDE)) {
3007                                         text = text.trim() + getTaggedCData(state, reader, next);
3008                                 } else if (isStartingElement(next, LIFE_CYCLE_PERIODS)) {
3009                                         handleNotYetImplementedElement(next);
3010                                 } else {
3011                                         handleUnexpectedStartElement(next.asStartElement());
3012                                 }
3013                         } else if (next.isCharacters()) {
3014                                 text += next.asCharacters().getData();
3015                         } else {
3016                                 handleUnexpectedElement(next);
3017                         }
3018                 }
3019                 throw new IllegalStateException("<Habitat> has no closing tag");
3020         }
3021
3022         private String getTaggedCData(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
3023                 checkNoAttributes(parentEvent);
3024
3025                 String text = getXmlTag(parentEvent);
3026                 while (reader.hasNext()) {
3027                         XMLEvent next = readNoWhitespace(reader);
3028                         if (isMyEndingElement(next, parentEvent)) {
3029                                 text += getXmlTag(next);
3030                                 return text;
3031                         } else if (next.isStartElement()) {
3032                                 text += getTaggedCData(state, reader, next);
3033                         } else if (next.isEndElement()) {
3034                                 text += getTaggedCData(state, reader, next);
3035                         } else if (next.isCharacters()) {
3036                                 text += next.asCharacters().getData();
3037                         } else {
3038                                 handleUnexpectedEndElement(next.asEndElement());
3039                         }
3040                 }
3041                 throw new IllegalStateException("Some tag has no closing tag");
3042         }
3043
3044         private String handleDistributionLocality(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent)throws XMLStreamException {
3045                 Map<String, Attribute> attributes = getAttributes(parentEvent);
3046                 String classValue = getAndRemoveRequiredAttributeValue(parentEvent, attributes, CLASS);
3047                 String statusValue =getAndRemoveAttributeValue(attributes, STATUS);
3048                 String frequencyValue =getAndRemoveAttributeValue(attributes, FREQUENCY);
3049
3050
3051                 Taxon taxon = state.getCurrentTaxon();
3052                 // TODO which ref to take?
3053                 Reference<?> ref = state.getConfig().getSourceReference();
3054
3055                 String text = "";
3056                 while (reader.hasNext()) {
3057                         XMLEvent next = readNoWhitespace(reader);
3058                         if (isMyEndingElement(next, parentEvent)) {
3059                                 if (StringUtils.isNotBlank(text)) {
3060                                         String label = CdmUtils.removeTrailingDot(normalize(text));
3061                                         TaxonDescription description = getTaxonDescription(taxon, ref, false, true);
3062                                         NamedAreaLevel level = makeNamedAreaLevel(state,classValue, next);
3063
3064                                         //status
3065                                         PresenceAbsenceTermBase<?> status = null;
3066                                         if (isNotBlank(statusValue)){
3067                                                 try {
3068                                                         status = state.getTransformer().getPresenceTermByKey(statusValue);
3069                                                         if (status == null){
3070                                                                 //TODO
3071                                                                 String message = "The status '%s' could not be transformed to an CDM status";
3072                                                                 fireWarningEvent(message, next, 4);
3073                                                         }
3074                                                 } catch (UndefinedTransformerMethodException e) {
3075                                                         throw new RuntimeException(e);
3076                                                 }
3077                                         }else{
3078                                                 status = PresenceTerm.PRESENT();
3079                                         }
3080                                         //frequency
3081                                         if (isNotBlank(frequencyValue)){
3082                                                 String message = "The frequency attribute is currently not yet available in CDM";
3083                                                 fireWarningEvent(message, parentEvent, 6);
3084                                         }
3085
3086                                         NamedArea higherArea = null;
3087                                         List<NamedArea> areas = new ArrayList<NamedArea>();
3088
3089                                         String patSingleArea = "([^,\\(]{3,})";
3090                                         String patSeparator = "(,|\\sand\\s)";
3091                                         String hierarchiePattern = String.format("%s\\((%s(%s%s)*)\\)",patSingleArea, patSingleArea, patSeparator, patSingleArea);
3092                                         Pattern patHierarchie = Pattern.compile(hierarchiePattern, Pattern.CASE_INSENSITIVE);
3093                                         Matcher matcher = patHierarchie.matcher(label);
3094                                         if (matcher.matches()){
3095                                                 String higherAreaStr = matcher.group(1).trim();
3096                                                 higherArea =  makeArea(state, higherAreaStr, level);
3097                                                 String[] innerAreas = matcher.group(2).split(patSeparator);
3098                                                 for (String innerArea : innerAreas){
3099                                                         if (isNotBlank(innerArea)){
3100                                                                 NamedArea singleArea = makeArea(state, innerArea.trim(), level);
3101                                                                 areas.add(singleArea);
3102                                                                 NamedArea partOf = singleArea.getPartOf();
3103 //                                                              if (partOf == null){
3104 //                                                                      singleArea.setPartOf(higherArea);
3105 //                                                              }
3106                                                         }
3107                                                 }
3108                                         }else{
3109                                                 NamedArea singleArea = makeArea(state, label, level);
3110                                                 areas.add(singleArea);
3111                                         }
3112
3113                                         for (NamedArea area : areas){
3114                                                 //create distribution
3115                                                 Distribution distribution = Distribution.NewInstance(area,status);
3116                                                 description.addElement(distribution);
3117                                         }
3118                                 } else {
3119                                         String message = "Empty distribution locality";
3120                                         fireWarningEvent(message, next, 4);
3121                                 }
3122                                 return text;
3123                         } else if (isStartingElement(next, COORDINATES)) {
3124                                 //TODO
3125                                 handleNotYetImplementedElement(next);
3126                         } else if (isEndingElement(next, COORDINATES)) {
3127                                 //TODO
3128                                 popUnimplemented(next.asEndElement());
3129                         } else if (next.isCharacters()) {
3130                                 text += next.asCharacters().getData();
3131                         } else {
3132                                 handleUnexpectedEndElement(next.asEndElement());
3133                         }
3134                 }
3135                 throw new IllegalStateException("<DistributionLocality> has no closing tag");
3136         }
3137
3138         /**
3139          * @param state
3140          * @param areaName
3141          * @param level
3142          * @return
3143          */
3144         private NamedArea makeArea(MarkupImportState state, String areaName, NamedAreaLevel level) {
3145
3146
3147                 //TODO FM vocabulary
3148                 TermVocabulary<NamedArea> voc = null;
3149                 NamedAreaType areaType = null;
3150
3151                 NamedArea area = null;
3152                 try {
3153                         area = state.getTransformer().getNamedAreaByKey(areaName);
3154                 } catch (UndefinedTransformerMethodException e) {
3155                         throw new RuntimeException(e);
3156                 }
3157                 if (area == null){
3158                         boolean isNewInState = false;
3159                         UUID uuid = state.getAreaUuid(areaName);
3160                         if (uuid == null){
3161                                 isNewInState = true;
3162
3163
3164                                 try {
3165                                         uuid = state.getTransformer().getNamedAreaUuid(areaName);
3166                                 } catch (UndefinedTransformerMethodException e) {
3167                                         throw new RuntimeException(e);
3168                                 }
3169                         }
3170
3171                         CdmImportBase.TermMatchMode matchMode = CdmImportBase.TermMatchMode.UUID_LABEL;
3172                         area = getNamedArea(state, uuid, areaName, areaName, areaName, areaType, level, voc, matchMode);
3173                         if (isNewInState){
3174                                 state.putAreaUuid(areaName, area.getUuid());
3175
3176                                 //TODO just for testing -> make generic and move to better place
3177                                 String geoServiceLayer="vmap0_as_bnd_political_boundary_a";
3178                                 String layerFieldName ="nam";
3179
3180                                 if ("Bangka".equals(areaName)){
3181                                         String areaValue = "PULAU BANGKA#SUMATERA SELATAN";
3182                                         GeoServiceArea geoServiceArea = new GeoServiceArea();
3183                                         geoServiceArea.add(geoServiceLayer, layerFieldName, areaValue);
3184                                         this.editGeoService.setMapping(area, geoServiceArea);
3185 //                                      save(area, state);
3186                                 }
3187                                 if ("Luzon".equals(areaName)){
3188                                         GeoServiceArea geoServiceArea = new GeoServiceArea();
3189
3190                                         List<String> list = Arrays.asList("HERMANA MAYOR ISLAND#CENTRAL LUZON",
3191                                                         "HERMANA MENOR ISLAND#CENTRAL LUZON",
3192                                                         "CENTRAL LUZON");
3193                                         for (String areaValue : list){
3194                                                 geoServiceArea.add(geoServiceLayer, layerFieldName, areaValue);
3195                                         }
3196
3197                                         this.editGeoService.setMapping(area, geoServiceArea);
3198 //                                      save(area, state);
3199                                 }
3200                                 if ("Mindanao".equals(areaName)){
3201                                         GeoServiceArea geoServiceArea = new GeoServiceArea();
3202
3203                                         List<String> list = Arrays.asList("NORTHERN MINDANAO",
3204                                                         "SOUTHERN MINDANAO",
3205                                                         "WESTERN MINDANAO");
3206                                         //TODO to be continued
3207                                         for (String areaValue : list){
3208                                                 geoServiceArea.add(geoServiceLayer, layerFieldName, areaValue);
3209                                         }
3210
3211                                         this.editGeoService.setMapping(area, geoServiceArea);
3212 //                                      save(area, state);
3213                                 }
3214                                 if ("Palawan".equals(areaName)){
3215                                         GeoServiceArea geoServiceArea = new GeoServiceArea();
3216
3217                                         List<String> list = Arrays.asList("PALAWAN#SOUTHERN TAGALOG");
3218                                         for (String areaValue : list){
3219                                                 geoServiceArea.add(geoServiceLayer, layerFieldName, areaValue);
3220                                         }
3221
3222                                         this.editGeoService.setMapping(area, geoServiceArea);
3223 //                                      save(area, state);
3224                                 }
3225
3226
3227                         }
3228                 }
3229                 return area;
3230         }
3231
3232
3233         /**
3234          * @param state
3235          * @param levelString
3236          * @param next
3237          * @return
3238          */
3239         private NamedAreaLevel makeNamedAreaLevel(MarkupImportState state,
3240                         String levelString, XMLEvent next) {
3241                 NamedAreaLevel level;
3242                 try {
3243                         level = state.getTransformer().getNamedAreaLevelByKey(levelString);
3244                         if (level == null) {
3245                                 UUID levelUuid = state.getTransformer().getNamedAreaLevelUuid(levelString);
3246                                 if (levelUuid == null) {
3247                                         String message = "Unknown distribution locality class (named area level): %s. Create new level instead.";
3248                                         message = String.format(message, levelString);
3249                                         fireWarningEvent(message, next, 6);
3250                                 }
3251                                 level = getNamedAreaLevel(state, levelUuid, levelString, levelString, levelString, null);
3252                         }
3253                 } catch (UndefinedTransformerMethodException e) {
3254                         throw new RuntimeException(e);
3255                 }
3256                 return level;
3257         }
3258
3259         private String handleHeading(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent)throws XMLStreamException {
3260                 checkNoAttributes(parentEvent);
3261
3262                 String text = "";
3263                 while (reader.hasNext()) {
3264                         XMLEvent next = readNoWhitespace(reader);
3265                         if (isMyEndingElement(next, parentEvent)) {
3266                                 return text;
3267                         } else if (next.isStartElement()) {
3268                                 if (isStartingElement(next, FOOTNOTE)) {
3269                                         handleNotYetImplementedElement(next);
3270                                 } else {
3271                                         handleUnexpectedStartElement(next.asStartElement());
3272                                 }
3273                         } else if (next.isCharacters()) {
3274                                 text += next.asCharacters().getData();
3275                         } else {
3276                                 handleUnexpectedEndElement(next.asEndElement());
3277                         }
3278                 }
3279                 throw new IllegalStateException("<String> has no closing tag");
3280
3281         }
3282
3283         /**
3284          * Handle string
3285          * @param state
3286          * @param reader
3287          * @param parentEvent
3288          * @param feature only needed for distributionLocalities
3289          * @return
3290          * @throws XMLStreamException
3291          */
3292         private Map<String, String> handleString(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent, Feature feature)throws XMLStreamException {
3293                 // attributes
3294                 String classValue = getClassOnlyAttribute(parentEvent, false);
3295                 if (StringUtils.isNotBlank(classValue)) {
3296                         String message = "class attribute for <string> not yet implemented";
3297                         fireWarningEvent(message, parentEvent, 2);
3298                 }
3299
3300                 // subheadings
3301                 Map<String, String> subHeadingMap = new HashMap<String, String>();
3302                 String currentSubheading = null;
3303
3304                 boolean isTextMode = true;
3305                 String text = "";
3306                 while (reader.hasNext()) {
3307                         XMLEvent next = readNoWhitespace(reader);
3308                         if (isMyEndingElement(next, parentEvent)) {
3309                                 putCurrentSubheading(subHeadingMap, currentSubheading, text);
3310                                 return subHeadingMap;
3311                         } else if (isStartingElement(next, BR)) {
3312                                 text += "<br/>";
3313                                 isTextMode = false;
3314                         } else if (isEndingElement(next, BR)) {
3315                                 isTextMode = true;
3316                         } else if (isHtml(next)) {
3317                                 text += getXmlTag(next);
3318                         } else if (isStartingElement(next, SUB_HEADING)) {
3319                                 text = putCurrentSubheading(subHeadingMap,currentSubheading, text);
3320                                 // TODO footnotes
3321                                 currentSubheading = getCData(state, reader, next).trim();
3322                         } else if (isStartingElement(next, DISTRIBUTION_LOCALITY)) {
3323                                 if (feature != null && !feature.equals(Feature.DISTRIBUTION())) {
3324                                         String message = "Distribution locality only allowed for feature of type 'distribution'";
3325                                         fireWarningEvent(message, next, 4);
3326                                 }
3327                                 text += handleDistributionLocality(state, reader, next);
3328                         } else if (next.isCharacters()) {
3329                                 if (!isTextMode) {
3330                                         String message = "String is not in text mode";
3331                                         fireWarningEvent(message, next, 6);
3332                                 } else {
3333                                         text += next.asCharacters().getData();
3334                                 }
3335                         } else if (isStartingElement(next, HEADING)) {
3336                                 //TODO
3337                                 handleNotYetImplementedElement(next);
3338                         } else if (isEndingElement(next, HEADING)) {
3339                                 //TODO
3340                                 popUnimplemented(next.asEndElement());
3341                         } else if (isStartingElement(next, QUOTE)) {
3342                                 //TODO
3343                                 handleNotYetImplementedElement(next);
3344                         } else if (isEndingElement(next, QUOTE)) {
3345                                 //TODO
3346                                 popUnimplemented(next.asEndElement());
3347                         } else if (isStartingElement(next, DEDICATION)) {
3348                                 //TODO
3349                                 handleNotYetImplementedElement(next);
3350                         } else if (isEndingElement(next, DEDICATION)) {
3351                                 //TODO
3352                                 popUnimplemented(next.asEndElement());
3353                         } else if (isStartingElement(next, TAXONTYPE)) {
3354                                 //TODO
3355                                 handleNotYetImplementedElement(next);
3356                         } else if (isEndingElement(next, TAXONTYPE)) {
3357                                 //TODO
3358                                 popUnimplemented(next.asEndElement());
3359                         } else if (isStartingElement(next, FULL_NAME)) {
3360                                 //TODO
3361                                 handleNotYetImplementedElement(next);
3362                         } else if (isEndingElement(next, FULL_NAME)) {
3363                                 //TODO
3364                                 popUnimplemented(next.asEndElement());
3365                         }else if (isStartingElement(next, REFERENCES)) {
3366                                 //TODO
3367                                 handleNotYetImplementedElement(next);
3368                         } else if (isEndingElement(next, REFERENCES)) {
3369                                 //TODO
3370                                 popUnimplemented(next.asEndElement());
3371                         } else if (isStartingElement(next, GATHERING)) {
3372                                 //TODO
3373                                 handleNotYetImplementedElement(next);
3374                         } else if (isEndingElement(next, GATHERING)) {
3375                                 //TODO
3376                                 popUnimplemented(next.asEndElement());
3377                         } else if (isStartingElement(next, ANNOTATION)) {
3378                                 //TODO
3379                                 handleNotYetImplementedElement(next);
3380                         } else if (isEndingElement(next, ANNOTATION)) {
3381                                 //TODO
3382                                 popUnimplemented(next.asEndElement());
3383                         } else if (isStartingElement(next, HABITAT)) {
3384                                 //TODO
3385                                 handleNotYetImplementedElement(next);
3386                         } else if (isEndingElement(next, HABITAT)) {
3387                                 //TODO
3388                                 popUnimplemented(next.asEndElement());
3389                         } else if (isStartingElement(next, FIGURE_REF)) {
3390                                 //TODO
3391                                 handleNotYetImplementedElement(next);
3392                         } else if (isEndingElement(next, FIGURE_REF)) {
3393                                 //TODO
3394                                 popUnimplemented(next.asEndElement());
3395                         } else if (isStartingElement(next, FIGURE)) {
3396                                 //TODO
3397                                 handleNotYetImplementedElement(next);
3398                         } else if (isEndingElement(next, FIGURE)) {
3399                                 //TODO
3400                                 popUnimplemented(next.asEndElement());
3401                         } else if (isStartingElement(next, FOOTNOTE_REF)) {
3402                                 //TODO
3403                                 handleNotYetImplementedElement(next);
3404                         } else if (isEndingElement(next, FOOTNOTE_REF)) {
3405                                 //TODO
3406                                 popUnimplemented(next.asEndElement());
3407                         } else if (isStartingElement(next, FOOTNOTE)) {
3408                                 //TODO
3409                                 handleNotYetImplementedElement(next);
3410                         } else if (isEndingElement(next, FOOTNOTE)) {
3411                                 //TODO
3412                                 popUnimplemented(next.asEndElement());
3413                         } else if (isStartingElement(next, WRITER)) {
3414                                 //TODO
3415                                 handleNotYetImplementedElement(next);
3416                         } else if (isEndingElement(next, WRITER)) {
3417                                 //TODO
3418                                 popUnimplemented(next.asEndElement());
3419                         } else if (isStartingElement(next, DATES)) {
3420                                 //TODO
3421                                 handleNotYetImplementedElement(next);
3422                         } else if (isEndingElement(next, DATES)) {
3423                                 //TODO
3424                                 popUnimplemented(next.asEndElement());
3425                         } else {
3426                                 handleUnexpectedElement(next);
3427                         }
3428                 }
3429                 throw new IllegalStateException("<String> has no closing tag");
3430         }
3431
3432         /**
3433          * @param subHeadingMap
3434          * @param currentSubheading
3435          * @param text
3436          * @return
3437          */
3438         private String putCurrentSubheading(Map<String, String> subHeadingMap, String currentSubheading, String text) {
3439                 if (StringUtils.isNotBlank(text)) {
3440                         text = removeStartingMinus(text);
3441                         subHeadingMap.put(currentSubheading, text.trim());
3442                 }
3443                 return "";
3444         }
3445
3446         private String removeStartingMinus(String string) {
3447                 string = replaceStart(string, "-");
3448                 string = replaceStart(string, "\u002d");
3449                 string = replaceStart(string, "\u2013");
3450                 string = replaceStart(string, "\u2014");
3451                 string = replaceStart(string, "--");
3452                 return string;
3453         }
3454
3455         /**
3456          * @param value
3457          * @param replacementString
3458          */
3459         private String replaceStart(String value, String replacementString) {
3460                 if (value.startsWith(replacementString) ){
3461                         value = value.substring(replacementString.length()).trim();
3462                 }
3463                 while (value.startsWith("-") || value.startsWith("\u2014") ){
3464                         value = value.substring("-".length()).trim();
3465                 }
3466                 return value;
3467         }
3468
3469         private String getXmlTag(XMLEvent event) {
3470                 String result;
3471                 if (event.isStartElement()) {
3472                         result = "<" + event.asStartElement().getName().getLocalPart()
3473                                         + ">";
3474                 } else if (event.isEndElement()) {
3475                         result = "</" + event.asEndElement().getName().getLocalPart() + ">";
3476                 } else {
3477                         String message = "Only start or end elements are allowed as Html tags";
3478                         throw new IllegalStateException(message);
3479                 }
3480                 return result;
3481         }
3482
3483         protected static final List<String> htmlList = Arrays.asList("sub", "sup",
3484                         "ol", "ul", "li", "i", "b", "table", "br","tr","td");
3485
3486         private boolean isHtml(XMLEvent event) {
3487                 if (event.isStartElement()) {
3488                         String tag = event.asStartElement().getName().getLocalPart();
3489                         return htmlList.contains(tag);
3490                 } else if (event.isEndElement()) {
3491                         String tag = event.asEndElement().getName().getLocalPart();
3492                         return htmlList.contains(tag);
3493                 } else {
3494                         return false;
3495                 }
3496
3497         }
3498
3499         private TextData handleChar(MarkupImportState state, XMLEventReader reader,
3500                         XMLEvent parentEvent) throws XMLStreamException {
3501                 String classValue = getClassOnlyAttribute(parentEvent);
3502                 Feature feature = makeFeature(classValue, state, parentEvent);
3503
3504                 boolean isTextMode = true;
3505                 String text = "";
3506                 while (reader.hasNext()) {
3507                         XMLEvent next = readNoWhitespace(reader);
3508                         if (isMyEndingElement(next, parentEvent)) {
3509                                 TextData textData = TextData.NewInstance(feature);
3510                                 textData.putText(Language.DEFAULT(), text);
3511                                 return textData;
3512                         } else if (isStartingElement(next, FIGURE_REF)) {
3513                                 //TODO
3514                                 handleNotYetImplementedElement(next);
3515                         } else if (isEndingElement(next, FIGURE_REF)) {
3516                                 //TODO
3517                                 popUnimplemented(next.asEndElement());
3518                         } else if (isStartingElement(next, FOOTNOTE_REF)) {
3519                                 //TODO
3520                                 handleNotYetImplementedElement(next);
3521                         } else if (isEndingElement(next, FOOTNOTE_REF)) {
3522                                 //TODO
3523                                 popUnimplemented(next.asEndElement());
3524                         } else if (isStartingElement(next, BR)) {
3525                                 text += "<br/>";
3526                                 isTextMode = false;
3527                         } else if (isEndingElement(next, BR)) {
3528                                 isTextMode = true;
3529                         } else if (isHtml(next)) {
3530                                 text += getXmlTag(next);
3531                         } else if (next.isStartElement()) {
3532                                 if (isStartingElement(next, ANNOTATION)) {
3533                                         handleNotYetImplementedElement(next);
3534                                 } else if (isStartingElement(next, ITALICS)) {
3535                                         handleNotYetImplementedElement(next);
3536                                 } else if (isStartingElement(next, BOLD)) {
3537                                         handleNotYetImplementedElement(next);
3538                                 } else if (isStartingElement(next, FIGURE)) {
3539                                         handleFigure(state, reader, next);
3540                                 } else if (isStartingElement(next, FOOTNOTE)) {
3541                                         FootnoteDataHolder footnote = handleFootnote(state, reader,     next);
3542                                         if (footnote.isRef()) {
3543                                                 String message = "Ref footnote not implemented here";
3544                                                 fireWarningEvent(message, next, 4);
3545                                         } else {
3546                                                 registerGivenFootnote(state, footnote);
3547                                         }
3548                                 } else {
3549                                         handleUnexpectedStartElement(next.asStartElement());
3550                                 }
3551                         } else if (next.isCharacters()) {
3552                                 if (!isTextMode) {
3553                                         String message = "String is not in text mode";
3554                                         fireWarningEvent(message, next, 6);
3555                                 } else {
3556                                         text += next.asCharacters().getData();
3557                                 }
3558                         } else {
3559                                 handleUnexpectedEndElement(next.asEndElement());
3560                         }
3561                 }
3562                 throw new IllegalStateException("RefPart has no closing tag");
3563         }
3564
3565         /**
3566          * @param classValue
3567          * @param state
3568          * @param parentEvent
3569          * @return
3570          * @throws UndefinedTransformerMethodException
3571          */
3572         private Feature makeFeature(String classValue, MarkupImportState state, XMLEvent parentEvent) {
3573                 UUID uuid;
3574                 try {
3575                         Feature feature = state.getTransformer().getFeatureByKey(classValue);
3576                         if (feature != null) {
3577                                 return feature;
3578                         }
3579                         uuid = state.getTransformer().getFeatureUuid(classValue);
3580                         if (uuid == null) {
3581                                 // TODO
3582                                 String message = "Uuid is not defined for '%s'";
3583                                 message = String.format(message, classValue);
3584                                 fireWarningEvent(message, parentEvent, 8);
3585                         }
3586                         String featureText = StringUtils.capitalize(classValue);
3587
3588                         // TODO eFlora vocabulary
3589                         TermVocabulary<Feature> voc = null;
3590                         feature = getFeature(state, uuid, featureText, featureText, classValue, voc);
3591                         if (feature == null) {
3592                                 throw new NullPointerException(classValue + " not recognized as a feature");
3593                         }
3594                         return feature;
3595                 } catch (Exception e) {
3596                         String message = "Could not create feature for %s: %s";
3597                         message = String.format(message, classValue, e.getMessage());
3598                         fireWarningEvent(message, parentEvent, 4);
3599                         return Feature.UNKNOWN();
3600                 }
3601         }
3602
3603
3604
3605 }