cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupDocumentImport.java

   1 /**
   2  * Copyright (C) 2009 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.markup;
  11
  12 import java.net.MalformedURLException;
  13 import java.net.URL;
  14 import java.util.ArrayList;
  15 import java.util.Arrays;
  16 import java.util.HashMap;
  17 import java.util.HashSet;
  18 import java.util.LinkedList;
  19 import java.util.List;
  20 import java.util.Map;
  21 import java.util.Queue;
  22 import java.util.Set;
  23 import java.util.UUID;
  24 import java.util.regex.Matcher;
  25 import java.util.regex.Pattern;
  26
  27 import javax.xml.stream.FactoryConfigurationError;
  28 import javax.xml.stream.Location;
  29 import javax.xml.stream.XMLEventReader;
  30 import javax.xml.stream.XMLStreamException;
  31 import javax.xml.stream.events.Attribute;
  32 import javax.xml.stream.events.StartElement;
  33 import javax.xml.stream.events.XMLEvent;
  34
  35 import org.apache.commons.lang.StringUtils;
  36 import org.apache.log4j.Logger;
  37 import org.springframework.beans.factory.annotation.Autowired;
  38 import org.springframework.security.access.PermissionEvaluator;
  39 import org.springframework.security.authentication.AuthenticationManager;
  40 import org.springframework.security.core.Authentication;
  41 import org.springframework.stereotype.Component;
  42 import org.springframework.transaction.TransactionStatus;
  43
  44 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  45 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
  46 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacadeCacheStrategy;
  47 import eu.etaxonomy.cdm.common.CdmUtils;
  48 import eu.etaxonomy.cdm.ext.geo.GeoServiceArea;
  49 import eu.etaxonomy.cdm.ext.geo.IEditGeoService;
  50 import eu.etaxonomy.cdm.io.common.ICdmIO;
  51 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  52 import eu.etaxonomy.cdm.io.markup.UnmatchedLeads.UnmatchedLeadsKey;
  53 import eu.etaxonomy.cdm.model.agent.AgentBase;
  54 import eu.etaxonomy.cdm.model.agent.INomenclaturalAuthor;
  55 import eu.etaxonomy.cdm.model.agent.Team;
  56 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  57 import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
  58 import eu.etaxonomy.cdm.model.common.Annotation;
  59 import eu.etaxonomy.cdm.model.common.AnnotationType;
  60 import eu.etaxonomy.cdm.model.common.CdmBase;
  61 import eu.etaxonomy.cdm.model.common.Extension;
  62 import eu.etaxonomy.cdm.model.common.ExtensionType;
  63 import eu.etaxonomy.cdm.model.common.Language;
  64 import eu.etaxonomy.cdm.model.common.TermVocabulary;
  65 import eu.etaxonomy.cdm.model.common.TimePeriod;
  66 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  67 import eu.etaxonomy.cdm.model.description.Distribution;
  68 import eu.etaxonomy.cdm.model.description.Feature;
  69 import eu.etaxonomy.cdm.model.description.KeyStatement;
  70 import eu.etaxonomy.cdm.model.description.PolytomousKey;
  71 import eu.etaxonomy.cdm.model.description.PolytomousKeyNode;
  72 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTermBase;
  73 import eu.etaxonomy.cdm.model.description.PresenceTerm;
  74 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  75 import eu.etaxonomy.cdm.model.description.TextData;
  76 import eu.etaxonomy.cdm.model.location.NamedArea;
  77 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  78 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  79 import eu.etaxonomy.cdm.model.media.IdentifiableMediaEntity;
  80 import eu.etaxonomy.cdm.model.media.Media;
  81 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  82 import eu.etaxonomy.cdm.model.name.HomotypicalGroup;
  83 import eu.etaxonomy.cdm.model.name.NameTypeDesignationStatus;
  84 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  85 import eu.etaxonomy.cdm.model.name.NomenclaturalStatus;
  86 import eu.etaxonomy.cdm.model.name.NomenclaturalStatusType;
  87 import eu.etaxonomy.cdm.model.name.NonViralName;
  88 import eu.etaxonomy.cdm.model.name.Rank;
  89 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  90 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  91 import eu.etaxonomy.cdm.model.occurrence.Collection;
  92 import eu.etaxonomy.cdm.model.occurrence.DerivedUnitBase;
  93 import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
  94 import eu.etaxonomy.cdm.model.occurrence.Specimen;
  95 import eu.etaxonomy.cdm.model.reference.IArticle;
  96 import eu.etaxonomy.cdm.model.reference.IJournal;
  97 import eu.etaxonomy.cdm.model.reference.Reference;
  98 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  99 import eu.etaxonomy.cdm.model.reference.ReferenceType;
 100 import eu.etaxonomy.cdm.model.taxon.Classification;
 101 import eu.etaxonomy.cdm.model.taxon.SynonymRelationshipType;
 102 import eu.etaxonomy.cdm.model.taxon.Taxon;
 103 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
 104 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
 105 import eu.etaxonomy.cdm.strategy.parser.NameTypeParser;
 106 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
 107 import eu.etaxonomy.cdm.strategy.parser.SpecimenTypeParser;
 108 import eu.etaxonomy.cdm.strategy.parser.SpecimenTypeParser.TypeInfo;
 109
 110 /**
 111  * @author a.mueller
 112  *
 113  */
 114 @Component
 115 public class MarkupDocumentImport extends MarkupImportBase implements ICdmIO<MarkupImportState> {
 116         private static final Logger logger = Logger.getLogger(MarkupDocumentImport.class);
 117
 118         private static final boolean CREATE_NEW = true;
 119         private static final boolean IS_IMAGE_GALLERY = true;
 120         private static final boolean NO_IMAGE_GALLERY = false;
 121
 122
 123         private static final String ACCEPTED = "accepted";
 124         private static final String ACCEPTED_NAME = "acceptedName";
 125         private static final String ADDENDA = "addenda";
 126         private static final String ALTERNATEPUBTITLE = "alternatepubtitle";
 127         private static final String ALTERNATIVE_COLLECTION_TYPE_STATUS = "alternativeCollectionTypeStatus";
 128         private static final String ALTERNATIVE_COLLECTOR = "alternativeCollector";
 129         private static final String ALTERNATIVE_FIELD_NUM = "alternativeFieldNum";
 130         private static final String ALTITUDE = "altitude";
 131         private static final String ANNOTATION = "annotation";
 132         private static final String AUTHOR = "author";
 133         private static final String BIBLIOGRAPHY = "bibliography";
 134         private static final String BIOGRAPHIES = "biographies";
 135         private static final String BOLD = "bold";
 136         private static final String BR = "br";
 137         private static final String CHAR = "char";
 138         private static final String CITATION = "citation";
 139         private static final String COLLECTION_AND_TYPE = "collectionAndType";
 140         private static final String COLLECTION_TYPE_STATUS = "collectionTypeStatus";
 141         private static final String COLLECTOR = "collector";
 142         private static final String COORDINATES = "coordinates";
 143         private static final String COUPLET = "couplet";
 144         private static final String DATES = "dates";
 145         private static final String DEDICATION = "dedication";
 146         private static final String DEFAULT_MEDIA_URL = "defaultMediaUrl";
 147         private static final String DESTROYED = "destroyed";
 148         private static final String DETAILS = "details";
 149         private static final String DISTRIBUTION_LIST = "distributionList";
 150         private static final String DISTRIBUTION_LOCALITY = "distributionLocality";
 151         private static final String EDITION = "edition";
 152         private static final String EDITORS = "editors";
 153         private static final String FEATURE = "feature";
 154         private static final String FIGURE = "figure";
 155         private static final String FIGURE_LEGEND = "figureLegend";
 156         private static final String FIGURE_PART = "figurePart";
 157         private static final String FIGURE_REF = "figureRef";
 158         private static final String FIGURE_TITLE = "figureTitle";
 159         private static final String FOOTNOTE = "footnote";
 160         private static final String FOOTNOTE_REF = "footnoteRef";
 161         private static final String FOOTNOTE_STRING = "footnoteString";
 162         private static final String FIELD_NUM = "fieldNum";
 163         private static final String FREQUENCY = "frequency";
 164         private static final String FULL_NAME = "fullName";
 165         private static final String FULL_TYPE = "fullType";
 166         private static final String GATHERING = "gathering";
 167         private static final String HEADING = "heading";
 168         private static final String HABITAT = "habitat";
 169         private static final String HABITAT_LIST = "habitatList";
 170         private static final String HOMONYM = "homonym";
 171         private static final String HOMOTYPES = "homotypes";
 172         private static final String ID = "id";
 173         private static final String INFRANK = "infrank";
 174         private static final String INFRAUT = "infraut";
 175         private static final String INFRPARAUT = "infrparaut";
 176         private static final String IS_SPOTCHARACTERS = "isSpotcharacters";
 177         private static final String ISSUE = "issue";
 178         private static final String ITALICS = "italics";
 179         private static final String KEY = "key";
 180         private static final String KEY_TITLE = "keyTitle";
 181         private static final String KEYNOTES = "keynotes";
 182         private static final String LIFE_CYCLE_PERIODS = "lifeCyclePeriods";
 183         private static final String LOCALITY = "locality";
 184         private static final String LOST = "lost";
 185         private static final String META_DATA = "metaData";
 186         private static final String NAME = "name";
 187         private static final String NAME_TYPE = "nameType";
 188         private static final String NOM = "nom";
 189         private static final String NOMENCLATURE = "nomenclature";
 190         private static final String NOT_FOUND = "notFound";
 191         private static final String NOT_SEEN = "notSeen";
 192         private static final String NOTES = "notes";
 193         private static final String NUM = "num";
 194         private static final String ORIGINAL_DETERMINATION = "originalDetermination";
 195         private static final String PAGES = "pages";
 196         private static final String PARAUT = "paraut";
 197         private static final String PUBFULLNAME = "pubfullname";
 198         private static final String PUBLICATION = "publication";
 199         private static final String PUBNAME = "pubname";
 200         private static final String PUBTITLE = "pubtitle";
 201         private static final String PUBTYPE = "pubtype";
 202         private static final String QUESTION = "question";
 203         private static final String QUOTE = "quote";
 204         private static final String RANK = "rank";
 205         private static final String REF = "ref";
 206         private static final String REF_NUM = "refNum";
 207         private static final String REF_PART = "refPart";
 208         private static final String REFERENCE = "reference";
 209         private static final String REFERENCES = "references";
 210         private static final String TAXON = "taxon";
 211         private static final String TAXONTITLE = "taxontitle";
 212         private static final String TAXONTYPE = "taxontype";
 213         private static final String TEXT = "text";
 214         private static final String TEXT_SECTION = "textSection";
 215         private static final String TO_COUPLET = "toCouplet";
 216         private static final String TO_KEY = "toKey";
 217         private static final String TO_TAXON = "toTaxon";
 218         private static final String TYPE = "type";
 219         private static final String TYPE_STATUS = "typeStatus";
 220         private static final String TREATMENT = "treatment";
 221         private static final String SERIALS_ABBREVIATIONS = "serialsAbbreviations";
 222         private static final String SPECIMEN_TYPE = "specimenType";
 223         private static final String STATUS = "status";
 224         private static final String STRING = "string";
 225         private static final String SUB_HEADING = "subHeading";
 226         private static final String SUB_COLLECTION = "subCollection";
 227         private static final String SYNONYM = "synonym";
 228         private static final String UNKNOWN = "unknown";
 229         private static final String URL = "url";
 230         private static final String USAGE = "usage";
 231         private static final String VOLUME = "volume";
 232         private static final String WRITER = "writer";
 233         private static final String YEAR = "year";
 234
 235         private NonViralNameParserImpl parser = new NonViralNameParserImpl();
 236
 237         // TODO make part of state, but state is renewed when invoking the import a
 238         // second time
 239         private UnmatchedLeads unmatchedLeads;
 240
 241         @Autowired
 242         private IEditGeoService editGeoService;
 243
 244         // TODO remove preliminary
 245         @Autowired
 246         private AuthenticationManager authenticationManager;
 247         private Authentication authentication;
 248         private PermissionEvaluator permissionEvaluator;
 249
 250         public MarkupDocumentImport() {
 251                 super();
 252                 System.out.println("TODO remove preliminary authentication");
 253                 // UsernamePasswordAuthenticationToken token = new
 254                 // UsernamePasswordAuthenticationToken("admin", "0000");
 255                 // authentication = authenticationManager.authenticate(token);
 256                 // SecurityContext context = SecurityContextHolder.getContext();
 257                 // context.setAuthentication(authentication);
 258                 // permissionEvaluator = new CdmPermissionEvaluator();
 259         }
 260
 261         @Override
 262         public boolean doCheck(MarkupImportState state) {
 263                 state.setCheck(true);
 264                 doInvoke(state);
 265                 state.setCheck(false);
 266                 return state.isSuccess();
 267         }
 268
 269         @Override
 270         public void doInvoke(MarkupImportState state) {
 271                 fireProgressEvent("Start import markup document", "Before start of document");
 272
 273                 Queue<CdmBase> outputStream = new LinkedList<CdmBase>();
 274
 275                 TransactionStatus tx = startTransaction();
 276                 // FIXME reset state
 277                 doAllTheOldOtherStuff(state);
 278
 279                 // START
 280                 try {
 281                         // StAX
 282                         XMLEventReader reader = getStaxReader(state);
 283                         state.setReader(reader);
 284                         // start document
 285                         if (!validateStartOfDocument(reader)) {
 286                                 state.setUnsuccessfull();
 287                                 return;
 288                         }
 289
 290                         // publication
 291                         String elName = PUBLICATION;
 292                         boolean hasPublication = false;
 293
 294                         while (reader.hasNext()) {
 295                                 XMLEvent nextEvent = reader.nextEvent();
 296                                 if (isStartingElement(nextEvent, elName)) {
 297                                         handlePublication(state, reader, nextEvent, elName);
 298                                         hasPublication = true;
 299                                 } else if (nextEvent.isEndDocument()) {
 300                                         if (!hasPublication) {
 301                                                 String message = "No publication root element found";
 302                                                 fireWarningEvent(message, nextEvent, 8);
 303                                         }
 304                                         // done
 305                                 } else {
 306                                         fireSchemaConflictEventExpectedStartTag(elName, reader);
 307                                 }
 308                         }
 309                         commitTransaction(tx);
 310
 311                         // //SAX
 312                         // ImportHandlerBase handler= new PublicationHandler(this);
 313                         // parseSAX(state, handler);
 314
 315                 } catch (FactoryConfigurationError e1) {
 316                         fireWarningEvent("Some error occurred while setting up xml factory. Data can't be imported", "Start", 16);
 317                         state.setUnsuccessfull();
 318                 } catch (XMLStreamException e1) {
 319                         fireWarningEvent("An XMLStreamException occurred while parsing. Data can't be imported", "Start", 16);
 320                         state.setUnsuccessfull();
 321                         // } catch (ParserConfigurationException e) {
 322                         // fireWarningEvent("A ParserConfigurationException occurred while parsing. Data can't be imported",
 323                         // "Start", 16);
 324                         // } catch (SAXException e) {
 325                         // fireWarningEvent("A SAXException occurred while parsing. Data can't be imported",
 326                         // "Start", 16);
 327                         // } catch (IOException e) {
 328                         // fireWarningEvent("An IO exception occurred while parsing. Data can't be imported",
 329                         // "Start", 16);
 330
 331                 }
 332
 333                 return;
 334
 335         }
 336
 337         private void handlePublication(MarkupImportState state, XMLEventReader reader, XMLEvent currentEvent,
 338                         String elName) throws XMLStreamException {
 339
 340                 // attributes
 341                 StartElement element = currentEvent.asStartElement();
 342                 Map<String, Attribute> attributes = getAttributes(element);
 343                 String lang = getAndRemoveAttributeValue(attributes, "lang");
 344                 if (lang != null){
 345                         Language language = getTermService().getLanguageByIso(lang);
 346                         state.setDefaultLanguage(language);
 347                 }
 348
 349                 handleUnexpectedAttributes(element.getLocation(), attributes, "noNamespaceSchemaLocation");
 350
 351                 while (reader.hasNext()) {
 352                         XMLEvent event = readNoWhitespace(reader);
 353                         // TODO cardinality of alternative
 354                         if (event.isEndElement()) {
 355                                 if (isEndingElement(event, elName)) {
 356                                         return;
 357                                 } else {
 358                                         if (isEndingElement(event, BIOGRAPHIES)) {
 359                                                 // NOT YET IMPLEMENTED
 360                                                 popUnimplemented(event.asEndElement());
 361                                         } else if (isEndingElement(event, REFERENCES)) {
 362                                                 // NOT YET IMPLEMENTED
 363                                                 popUnimplemented(event.asEndElement());
 364                                         } else if (isEndingElement(event, TEXT_SECTION)) {
 365                                                 // NOT YET IMPLEMENTED
 366                                                 popUnimplemented(event.asEndElement());
 367                                         } else if (isEndingElement(event, ADDENDA)) {
 368                                                 // NOT YET IMPLEMENTED
 369                                                 popUnimplemented(event.asEndElement());
 370                                         } else {
 371                                                 handleUnexpectedElement(event);
 372                                         }
 373                                 }
 374                         } else if (event.isStartElement()) {
 375                                 if (isStartingElement(event, META_DATA)) {
 376                                         handleMetaData(state, reader, event);
 377                                 } else if (isStartingElement(event, TREATMENT)) {
 378                                         handleTreatment(state, reader, event);
 379                                 } else if (isStartingElement(event, BIOGRAPHIES)) {
 380                                         handleNotYetImplementedElement(event);
 381                                 } else if (isStartingElement(event, REFERENCES)) {
 382                                         handleNotYetImplementedElement(event);
 383                                 } else if (isStartingElement(event, TEXT_SECTION)) {
 384                                         handleNotYetImplementedElement(event);
 385                                 } else if (isStartingElement(event, ADDENDA)) {
 386                                         handleNotYetImplementedElement(event);
 387                                 } else {
 388                                         handleUnexpectedStartElement(event);
 389                                 }
 390                         } else {
 391                                 handleUnexpectedElement(event);
 392                         }
 393                 }
 394                 throw new IllegalStateException("Publication has no ending element");
 395         }
 396
 397         private void handleMetaData(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
 398                 checkNoAttributes(parentEvent);
 399
 400                 while (reader.hasNext()) {
 401                         XMLEvent next = readNoWhitespace(reader);
 402                         if (isMyEndingElement(next, parentEvent)) {
 403                                 return;
 404                         } else if (isStartingElement(next, DEFAULT_MEDIA_URL)) {
 405                                 String baseUrl = getCData(state, reader, next);
 406                                 try {
 407                                         new URL(baseUrl);
 408                                         state.setBaseMediaUrl(baseUrl);
 409                                 } catch (MalformedURLException e) {
 410                                         String message = "defaultMediaUrl '%s' is not a valid URL";
 411                                         message = String.format(message, baseUrl);
 412                                         fireWarningEvent(message, next, 8);
 413                                 }
 414                         } else {
 415                                 handleUnexpectedElement(next);
 416                         }
 417                 }
 418                 throw new IllegalStateException("MetaData has no ending element");
 419
 420         }
 421
 422         private void handleTreatment(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
 423                 checkNoAttributes(parentEvent);
 424                 Taxon lastTaxon = null;
 425                 while (reader.hasNext()) {
 426                         XMLEvent next = readNoWhitespace(reader);
 427                         if (isStartingElement(next, TAXON)) {
 428                                 Taxon thisTaxon = handleTaxon(state, reader, next.asStartElement());
 429                                 doTaxonRelation(state, thisTaxon, lastTaxon, parentEvent.getLocation());
 430                                 lastTaxon = thisTaxon;
 431                                 // TODO for imports spanning multiple documents ?? Still needed?
 432                                 state.getConfig().setLastTaxonUuid(lastTaxon.getUuid());
 433                         } else if (isMyEndingElement(next, parentEvent)) {
 434                                 Set<PolytomousKeyNode> keyNodesToSave = state.getPolytomousKeyNodesToSave();
 435                                 //better save the key then the nodes
 436                                 Set<PolytomousKey> keySet = new HashSet<PolytomousKey>();
 437                                 for (PolytomousKeyNode node : keyNodesToSave){
 438                                         PolytomousKey key = node.getKey();
 439                                         keySet.add(key);
 440                                 }
 441                                 save(keySet, state);
 442                                 //unmatched key leads
 443                                 UnmatchedLeads unmatched = state.getUnmatchedLeads();
 444                                 if (unmatched.size() > 0){
 445                                         String message = "The following key leads are unmatched: %s";
 446                                         message = String.format(message, state.getUnmatchedLeads().toString());
 447                                         fireWarningEvent(message, next, 6);
 448                                 }
 449 //                              save(keyNodesToSave, state);
 450
 451                                 return;
 452                         } else {
 453                                 handleUnexpectedElement(next);
 454                         }
 455                 }
 456                 return;
 457         }
 458
 459         /**
 460          * @param taxon
 461          * @param lastTaxon
 462          */
 463         private void doTaxonRelation(MarkupImportState state, Taxon taxon,
 464                         Taxon lastTaxon, Location dataLocation) {
 465
 466                 Classification tree = makeTree(state, dataLocation);
 467                 if (lastTaxon == null) {
 468                         tree.addChildTaxon(taxon, null, null, null);
 469                         return;
 470                 }
 471                 Rank thisRank = taxon.getName().getRank();
 472                 Rank lastRank = lastTaxon.getName().getRank();
 473                 if (lastTaxon.getTaxonNodes().size() > 0) {
 474                         TaxonNode lastNode = lastTaxon.getTaxonNodes().iterator().next();
 475                         if (thisRank == null){
 476                                 String message = "rank is undefined for taxon '%s'. Can't create classification without rank.";
 477                                 message = String.format(message, taxon.getName().getTitleCache());
 478                                 fireWarningEvent(message, makeLocationStr(dataLocation), 6);
 479                         }else if (thisRank.isLower(lastRank)) {
 480                                 lastNode.addChildTaxon(taxon, null, null, null);
 481                                 fillMissingEpithetsForTaxa(lastTaxon, taxon);
 482                         } else if (thisRank.equals(lastRank)) {
 483                                 TaxonNode parent = lastNode.getParent();
 484                                 if (parent != null) {
 485                                         parent.addChildTaxon(taxon, null, null, null);
 486                                         fillMissingEpithetsForTaxa(parent.getTaxon(), taxon);
 487                                 } else {
 488                                         tree.addChildTaxon(taxon, null, null, null);
 489                                 }
 490                         } else if (thisRank.isHigher(lastRank)) {
 491                                 doTaxonRelation(state, taxon, lastNode.getParent().getTaxon(),  dataLocation);
 492                                 // TaxonNode parentNode = handleTaxonRelation(state, taxon,
 493                                 // lastNode.getParent().getTaxon());
 494                                 // parentNode.addChildTaxon(taxon, null, null, null);
 495                         }
 496                 } else {
 497
 498                         String message = "Last taxon has no node";
 499                         fireWarningEvent(message, makeLocationStr(dataLocation), 6);
 500                 }
 501         }
 502
 503         /**
 504          * @param state
 505          * @param dataLocation
 506          * @return
 507          */
 508         private Classification makeTree(MarkupImportState state, Location dataLocation) {
 509                 Classification result = state.getTree(null);
 510                 if (result == null) {
 511                         UUID uuid = state.getConfig().getClassificationUuid();
 512                         if (uuid == null) {
 513                                 String message = "No classification uuid is defined";
 514                                 fireWarningEvent(message, makeLocationStr(dataLocation), 6);
 515                                 result = createNewClassification(state);
 516                         } else {
 517                                 result = getClassificationService().find(uuid);
 518                                 if (result == null) {
 519                                         result = createNewClassification(state);
 520                                         result.setUuid(uuid);
 521                                 }
 522                         }
 523                         state.putTree(null, result);
 524                 }
 525                 save(result, state);
 526                 return result;
 527         }
 528
 529         private Classification createNewClassification(MarkupImportState state) {
 530                 Classification result;
 531                 result = Classification.NewInstance(state.getConfig().getClassificationTitle());
 532                 state.putTree(null, result);
 533                 return result;
 534         }
 535
 536         private Taxon handleTaxon(MarkupImportState state, XMLEventReader reader, StartElement parentEvent) throws XMLStreamException {
 537                 // TODO progress monitoring
 538                 Map<String, Attribute> attributes = getAttributes(parentEvent);
 539                 Taxon taxon = createTaxonAndName(state, attributes);
 540                 state.setCurrentTaxon(taxon);
 541
 542                 boolean hasTitle = false;
 543                 boolean hasNomenclature = false;
 544                 String taxonTitle = null;
 545
 546                 while (reader.hasNext()) {
 547                         XMLEvent next = readNoWhitespace(reader);
 548                         if (next.isEndElement()) {
 549                                 if (isMyEndingElement(next, parentEvent)) {
 550                                         checkMandatoryElement(hasTitle, parentEvent, TAXONTITLE);
 551                                         checkMandatoryElement(hasNomenclature, parentEvent,     NOMENCLATURE);
 552                                         handleUnexpectedAttributes(parentEvent.getLocation(),attributes);
 553
 554                                         makeKeyNodes(state, parentEvent, taxonTitle);
 555                                         state.setCurrentTaxon(null);
 556                                         state.setCurrentTaxonNum(null);
 557                                         save(taxon, state);
 558                                         return taxon;
 559                                 } else {
 560                                         if (isEndingElement(next, HEADING)) {
 561                                                 // NOT YET IMPLEMENTED
 562                                                 popUnimplemented(next.asEndElement());
 563                                         } else if (isEndingElement(next, TEXT_SECTION)) {
 564                                                 // NOT YET IMPLEMENTED
 565                                                 popUnimplemented(next.asEndElement());
 566                                         } else if (isEndingElement(next, REFERENCES)) {
 567                                                 // NOT YET IMPLEMENTED
 568                                                 popUnimplemented(next.asEndElement());
 569                                         } else if (isEndingElement(next, FIGURE_REF)) {
 570                                                 // NOT YET IMPLEMENTED
 571                                                 popUnimplemented(next.asEndElement());
 572                                         } else {
 573                                                 handleUnexpectedEndElement(next.asEndElement());
 574                                         }
 575                                 }
 576                         } else if (next.isStartElement()) {
 577                                 if (isStartingElement(next, HEADING)) {
 578                                         handleNotYetImplementedElement(next);
 579                                 } else if (isStartingElement(next, TAXONTITLE)) {
 580                                         taxonTitle = handleTaxonTitle(state, reader, next);
 581                                         hasTitle = true;
 582                                 } else if (isStartingElement(next, WRITER)) {
 583                                         makeKeyWriter(state, reader, taxon, taxonTitle, next);
 584                                 } else if (isStartingElement(next, TEXT_SECTION)) {
 585                                         handleNotYetImplementedElement(next);
 586                                 } else if (isStartingElement(next, KEY)) {
 587                                         handleKey(state, reader, next);
 588                                 } else if (isStartingElement(next, NOMENCLATURE)) {
 589                                         handleNomenclature(state, reader, next);
 590                                         hasNomenclature = true;
 591                                 } else if (isStartingElement(next, FEATURE)) {
 592                                         handleFeature(state, reader, next);
 593                                 } else if (isStartingElement(next, NOTES)) {
 594                                         // TODO is this the correct way to handle notes?
 595                                         String note = handleNotes(state, reader, next);
 596
 597                                         UUID notesUuid;
 598                                         try {
 599                                                 notesUuid = state.getTransformer().getFeatureUuid(
 600                                                                 "notes");
 601                                                 Feature feature = getFeature(state, notesUuid, "Notes",
 602                                                                 "Notes", "note", null);
 603                                                 TextData textData = TextData.NewInstance(feature);
 604                                                 textData.putText(Language.DEFAULT(), note);
 605                                                 TaxonDescription description = getTaxonDescription(
 606                                                                 taxon, false, true);
 607                                                 description.addElement(textData);
 608                                         } catch (UndefinedTransformerMethodException e) {
 609                                                 String message = "getFeatureUuid method not yet implemented";
 610                                                 fireWarningEvent(message, next, 8);
 611                                         }
 612                                 } else if (isStartingElement(next, REFERENCES)) {
 613                                         handleNotYetImplementedElement(next);
 614                                 } else if (isStartingElement(next, FIGURE_REF)) {
 615                                         handleNotYetImplementedElement(next);
 616                                 } else if (isStartingElement(next, FIGURE)) {
 617                                         handleFigure(state, reader, next);
 618                                 } else if (isStartingElement(next, FOOTNOTE)) {
 619                                         FootnoteDataHolder footnote = handleFootnote(state, reader,
 620                                                         next);
 621                                         if (footnote.isRef()) {
 622                                                 String message = "Ref footnote not implemented here";
 623                                                 fireWarningEvent(message, next, 4);
 624                                         } else {
 625                                                 registerGivenFootnote(state, footnote);
 626                                         }
 627                                 } else {
 628                                         handleUnexpectedStartElement(next);
 629                                 }
 630                         } else {
 631                                 handleUnexpectedElement(next);
 632                         }
 633                 }
 634                 throw new IllegalStateException("<Taxon> has no closing tag");
 635         }
 636
 637         private void makeKeyNodes(MarkupImportState state, XMLEvent event, String taxonTitle) {
 638                 Taxon taxon = state.getCurrentTaxon();
 639                 String num = state.getCurrentTaxonNum();
 640
 641                 String nameString = CdmBase.deproxy(taxon.getName(), NonViralName.class).getNameCache();
 642 //              String nameString = taxonTitle;
 643
 644                 //try to find matching lead nodes
 645                 UnmatchedLeadsKey leadsKey = UnmatchedLeadsKey.NewInstance(num, nameString);
 646                 Set<PolytomousKeyNode> matchingNodes = handleMatchingNodes(state, taxon, leadsKey);
 647
 648                 if (num != null){//same without using the num
 649                         UnmatchedLeadsKey noNumLeadsKey = UnmatchedLeadsKey.NewInstance("", nameString);
 650                         Set<PolytomousKeyNode> noNumMatchingNodes = handleMatchingNodes(state, taxon, noNumLeadsKey);
 651                         if(noNumMatchingNodes.size() > 0){
 652                                 String message ="Taxon matches additional key node when not considering <num> attribute in taxontitle. This may be correct but may also indicate an error.";
 653                                 fireWarningEvent(message, event, 1);
 654                         }
 655                 }
 656                 //report missing match, if num exists
 657                 if (matchingNodes.isEmpty() && num != null){
 658                         String message = "Taxon has <num> attribute in taxontitle but no matching key nodes exist: %s, Key: %s";
 659                         message = String.format(message, num, leadsKey.toString());
 660                         fireWarningEvent(message, event, 1);
 661                 }
 662
 663         }
 664
 665         private Set<PolytomousKeyNode> handleMatchingNodes(MarkupImportState state, Taxon taxon, UnmatchedLeadsKey leadsKey) {
 666                 Set<PolytomousKeyNode> matchingNodes = state.getUnmatchedLeads().getNodes(leadsKey);
 667                 for (PolytomousKeyNode matchingNode : matchingNodes){
 668                         state.getUnmatchedLeads().removeNode(leadsKey, matchingNode);
 669                         matchingNode.setTaxon(taxon);
 670                         state.getPolytomousKeyNodesToSave().add(matchingNode);
 671                 }
 672                 return matchingNodes;
 673         }
 674
 675         private void handleKey(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
 676                 // attributes
 677                 Map<String, Attribute> attributes = getAttributes(parentEvent);
 678                 String isSpotcharacters = getAndRemoveAttributeValue(attributes, IS_SPOTCHARACTERS);
 679                 if (isNotBlank(isSpotcharacters) ) {
 680                         //TODO isSpotcharacters
 681                         String message = "Attribute isSpotcharacters not yet implemented for <key>";
 682                         fireWarningEvent(message, parentEvent, 4);
 683                 }
 684
 685                 PolytomousKey key = PolytomousKey.NewInstance();
 686                 key.addTaxonomicScope(state.getCurrentTaxon());
 687                 state.setCurrentKey(key);
 688
 689                 boolean isFirstCouplet = true;
 690                 while (reader.hasNext()) {
 691                         XMLEvent next = readNoWhitespace(reader);
 692                         if (isMyEndingElement(next, parentEvent)) {
 693                                 save(key, state);
 694                                 state.setCurrentKey(null);
 695                                 return;
 696                         } else if (isEndingElement(next, KEYNOTES)){
 697                                 popUnimplemented(next.asEndElement());
 698                         } else if (isStartingElement(next, KEY_TITLE)) {
 699                                         handleKeyTitle(state, reader, next);
 700                         } else if (isStartingElement(next, KEYNOTES)) {
 701                                 //TODO
 702                                 handleNotYetImplementedElement(next);
 703                         } else if (isStartingElement(next, COUPLET)) {
 704                                 PolytomousKeyNode node = null;
 705                                 if (isFirstCouplet){
 706                                         node = key.getRoot();
 707                                         isFirstCouplet = false;
 708                                 }
 709                                 handleCouplet(state, reader, next, node);
 710                         } else {
 711                                 handleUnexpectedElement(next);
 712                         }
 713                 }
 714                 throw new IllegalStateException("<key> has no closing tag");
 715         }
 716
 717         /**
 718          * @param state
 719          * @param reader
 720          * @param key
 721          * @param next
 722          * @throws XMLStreamException
 723          */
 724         private void handleKeyTitle(MarkupImportState state, XMLEventReader reader, XMLEvent next) throws XMLStreamException {
 725                 PolytomousKey key = state.getCurrentKey();
 726                 String keyTitle = getCData(state, reader, next);
 727                 String standardTitles = "(?i)(Key\\sto\\sthe\\s(genera|species|varieties|forms))";
 728
 729                 if (isNotBlank(keyTitle) ){
 730                         if (!state.getConfig().isReplaceStandardKeyTitles() || ! keyTitle.matches(standardTitles)){
 731                                 key.setTitleCache(keyTitle, true);
 732                         }
 733                 }
 734         }
 735
 736         private void handleCouplet(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent, PolytomousKeyNode parentNode) throws XMLStreamException {
 737                 String num = getOnlyAttribute(parentEvent, NUM, true);
 738                 List<PolytomousKeyNode> childList = new ArrayList<PolytomousKeyNode>();
 739
 740                 while (reader.hasNext()) {
 741                         XMLEvent next = readNoWhitespace(reader);
 742                         if (isMyEndingElement(next, parentEvent)) {
 743                                 completeCouplet(state, parentEvent, parentNode, num, childList);
 744                                 return;
 745                         } else if (isStartingElement(next, QUESTION)) {
 746                                 handleQuestion(state, reader, next, childList);
 747                         } else if (isStartingElement(next, KEYNOTES)) {
 748                                 //TODO
 749                                 handleNotYetImplementedElement(next);
 750                         } else if (isEndingElement(next, KEYNOTES)) {
 751                                 //TODO
 752                                 popUnimplemented(next.asEndElement());
 753                         } else {
 754                                 handleUnexpectedElement(next);
 755                         }
 756                 }
 757                 throw new IllegalStateException("<couplet> has no closing tag");
 758         }
 759
 760         /**
 761          * @param state
 762          * @param parentEvent
 763          * @param parentNode
 764          * @param num
 765          * @param childList
 766          */
 767         private void completeCouplet(MarkupImportState state, XMLEvent parentEvent,
 768                         PolytomousKeyNode parentNode, String num,
 769                         List<PolytomousKeyNode> childList) {
 770                 if (parentNode != null){
 771                         for (PolytomousKeyNode childNode : childList){
 772                                 parentNode.addChild(childNode);
 773                         }
 774                 }else if (isNotBlank(num)){
 775                         UnmatchedLeadsKey unmatchedKey = UnmatchedLeadsKey.NewInstance(state.getCurrentKey(), num);
 776                         Set<PolytomousKeyNode> nodes = state.getUnmatchedLeads().getNodes(unmatchedKey);
 777                         for(PolytomousKeyNode nodeToMatch: nodes){
 778                                 for (PolytomousKeyNode childNode : childList){
 779                                         nodeToMatch.addChild(childNode);
 780                                 }
 781                                 state.getUnmatchedLeads().removeNode(unmatchedKey, nodeToMatch);
 782                         }
 783                 }else{
 784                         String message = "Parent num could not be matched. Please check if num (%s) is correct";
 785                         message = String.format(message, num);
 786                         fireWarningEvent(message, parentEvent, 6);
 787                 }
 788         }
 789
 790         private void handleQuestion(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent, List<PolytomousKeyNode> nodesList) throws XMLStreamException {
 791                 // attributes
 792                 Map<String, Attribute> attributes = getAttributes(parentEvent);
 793                 //needed only for data lineage
 794                 String questionNum = getAndRemoveRequiredAttributeValue(parentEvent, attributes, NUM);
 795
 796                 PolytomousKeyNode myNode = PolytomousKeyNode.NewInstance();
 797                 myNode.setKey(state.getCurrentKey());  //to avoid NPE while computing num in PolytomousKeyNode in case this node is not matched correctly with a parent
 798                 nodesList.add(myNode);
 799
 800                 while (reader.hasNext()) {
 801                         XMLEvent next = readNoWhitespace(reader);
 802                         if (isMyEndingElement(next, parentEvent)) {
 803                                 return;
 804                         } else if (isStartingElement(next, TEXT)) {
 805                                 String text = getCData(state, reader, next);
 806                                 KeyStatement statement = KeyStatement.NewInstance(text);
 807                                 myNode.setStatement(statement);
 808                         } else if (isStartingElement(next, COUPLET)) {
 809                                 //TODO test
 810                                 handleCouplet(state, reader, next, myNode);
 811                         } else if (isStartingElement(next, TO_COUPLET)) {
 812                                 handleToCouplet(state, reader, next, myNode);
 813                         } else if (isStartingElement(next, TO_TAXON)) {
 814                                 handleToTaxon(state, reader, next, myNode);
 815
 816                         } else if (isStartingElement(next, TO_KEY)) {
 817                                 //TODO
 818                                 handleNotYetImplementedElement(next);
 819                         } else if (isEndingElement(next, TO_KEY)){
 820                                 //TODO
 821                                 popUnimplemented(next.asEndElement());
 822                         } else if (isStartingElement(next, KEYNOTES)) {
 823                                 //TODO
 824                                 handleNotYetImplementedElement(next);
 825                         } else if (isEndingElement(next, KEYNOTES)){
 826                                 //TODO
 827                                 popUnimplemented(next.asEndElement());
 828                         } else {
 829                                 handleUnexpectedElement(next);
 830                         }
 831                 }
 832                 throw new IllegalStateException("<question> has no closing tag");
 833         }
 834
 835         private void handleToCouplet(MarkupImportState state, XMLEventReader reader, XMLEvent next, PolytomousKeyNode node) throws XMLStreamException {
 836                 String num = getOnlyAttribute(next, NUM, true);
 837                 String cData = getCData(state, reader, next, false);
 838                 if (isNotBlank(cData) && ! cData.equals(num)){
 839                         String message = "CData ('%s') not handled in <toCouplet>";
 840                         message = String.format(message, cData);
 841                         fireWarningEvent(message, next, 4);
 842                 }
 843                 UnmatchedLeadsKey unmatched = UnmatchedLeadsKey.NewInstance(state.getCurrentKey(), num);
 844                 state.getUnmatchedLeads().addKey(unmatched, node);
 845         }
 846
 847         private void handleToTaxon(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent, PolytomousKeyNode node) throws XMLStreamException {
 848                 Map<String, Attribute> attributes = getAttributes(parentEvent);
 849                 String num = getAndRemoveAttributeValue(attributes, NUM);
 850                 String taxonStr = getCData(state, reader, parentEvent, false);
 851                 //TODO ?
 852                 taxonStr = makeTaxonKey(taxonStr, state.getCurrentTaxon());
 853                 UnmatchedLeadsKey unmatched = UnmatchedLeadsKey.NewInstance(num, taxonStr);
 854                 state.getUnmatchedLeads().addKey(unmatched, node);
 855                 return;
 856         }
 857
 858         private String makeTaxonKey(String strGoto, Taxon taxon) {
 859                 String result = "";
 860                 if (strGoto == null){
 861                         return "";
 862                 }
 863
 864                 NonViralName<?> name = CdmBase.deproxy(taxon.getName(), NonViralName.class);
 865                 String strGenusName = name.getGenusOrUninomial();
 866
 867
 868                 strGoto = strGoto.replaceAll("\\([^\\(\\)]*\\)", "");  //replace all brackets
 869                 strGoto = strGoto.replaceAll("\\s+", " "); //replace multiple whitespaces by exactly one whitespace
 870
 871                 strGoto = strGoto.trim();
 872                 String[] split = strGoto.split("\\s");
 873                 for (int i = 0; i<split.length; i++){
 874                         String single = split[i];
 875                         if (isGenusAbbrev(single, strGenusName)){
 876                                 split[i] = strGenusName;
 877                         }
 878                         if (isInfraSpecificMarker(single)){
 879                                 String strSpeciesEpi = name.getSpecificEpithet();
 880                                 if (isBlank(result)){
 881                                         result += strGenusName + " " + strSpeciesEpi;
 882                                 }
 883                         }
 884                         result = (result + " " + split[i]).trim();
 885                 }
 886                 return result;
 887         }
 888
 889
 890         private boolean isInfraSpecificMarker(String single) {
 891                 try {
 892                         if (Rank.getRankByAbbreviation(single).isInfraSpecific()){
 893                                 return true;
 894                         }else{
 895                                 return false;
 896                         }
 897                 } catch (UnknownCdmTypeException e) {
 898                         return false;
 899                 }
 900         }
 901
 902         private boolean isGenusAbbrev(String single, String strGenusName) {
 903                 if (! single.matches("[A-Z]\\.?")) {
 904                         return false;
 905                 }else if (single.length() == 0 || strGenusName == null || strGenusName.length() == 0){
 906                         return false;
 907                 }else{
 908                         return single.charAt(0) == strGenusName.charAt(0);
 909                 }
 910         }
 911
 912         /**
 913          * @param state
 914          * @param reader
 915          * @param taxon
 916          * @param taxonTitle
 917          * @param next
 918          * @throws XMLStreamException
 919          */
 920         private void makeKeyWriter(MarkupImportState state, XMLEventReader reader, Taxon taxon, String taxonTitle, XMLEvent next) throws XMLStreamException {
 921                 WriterDataHolder writer = handleWriter(state, reader, next);
 922                 taxon.addExtension(writer.extension);
 923                 // TODO what if taxonTitle comes later
 924                 if (StringUtils.isNotBlank(taxonTitle)
 925                                 && writer.extension != null) {
 926                         Reference<?> sec = ReferenceFactory.newBookSection();
 927                         sec.setTitle(taxonTitle);
 928                         TeamOrPersonBase<?> author = createAuthor(writer.writer);
 929                         sec.setAuthorTeam(author);
 930                         sec.setInReference(state.getConfig()
 931                                         .getSourceReference());
 932                         taxon.setSec(sec);
 933                         registerFootnotes(state, sec, writer.footnotes);
 934                 } else {
 935                         String message = "No taxontitle exists for writer";
 936                         fireWarningEvent(message, next, 6);
 937                 }
 938         }
 939
 940         private String handleNotes(MarkupImportState state, XMLEventReader reader,
 941                         XMLEvent parentEvent) throws XMLStreamException {
 942                 checkNoAttributes(parentEvent);
 943
 944                 String text = "";
 945                 while (reader.hasNext()) {
 946                         XMLEvent next = readNoWhitespace(reader);
 947                         if (isMyEndingElement(next, parentEvent)) {
 948                                 return text;
 949                         } else if (next.isEndElement()) {
 950                                 if (isEndingElement(next, HEADING)) {
 951                                         popUnimplemented(next.asEndElement());
 952                                 } else if (isEndingElement(next, WRITER)) {
 953                                         popUnimplemented(next.asEndElement());
 954                                 } else if (isEndingElement(next, NUM)) {
 955                                         popUnimplemented(next.asEndElement());
 956                                 } else {
 957                                         handleUnexpectedEndElement(next.asEndElement());
 958                                 }
 959                         } else if (next.isStartElement()) {
 960                                 if (isStartingElement(next, HEADING)) {
 961                                         handleNotYetImplementedElement(next);
 962                                 } else if (isStartingElement(next, SUB_HEADING)) {
 963                                         String subheading = getCData(state, reader, next).trim();
 964                                         if (! isNoteHeading(subheading)) {
 965                                                 fireNotYetImplementedElement(next.getLocation(), next.asStartElement().getName(), 0);
 966                                         }
 967                                 } else if (isStartingElement(next, WRITER)) {
 968                                         handleNotYetImplementedElement(next);
 969                                 } else if (isStartingElement(next, NUM)) {
 970                                         handleNotYetImplementedElement(next);
 971                                 } else if (isStartingElement(next, STRING)) {
 972                                         // TODO why multiple strings in schema?
 973                                         text = makeNotesString(state, reader, text, next);
 974                                 } else {
 975                                         handleUnexpectedStartElement(next.asStartElement());
 976                                 }
 977                         } else {
 978                                 handleUnexpectedElement(next);
 979                         }
 980                 }
 981                 throw new IllegalStateException("<Notes> has no closing tag");
 982         }
 983
 984         /**
 985          * @param state
 986          * @param reader
 987          * @param text
 988          * @param next
 989          * @return
 990          * @throws XMLStreamException
 991          */
 992         private String makeNotesString(MarkupImportState state, XMLEventReader reader, String text, XMLEvent next) throws XMLStreamException {
 993                 Map<String, String> stringMap = handleString(state, reader,     next, null);
 994                 if (stringMap.size() == 0){
 995                         String message = "No text available in <notes>";
 996                         fireWarningEvent(message, next, 4);
 997                 }else if (stringMap.size() > 1){
 998                         String message = "Subheadings not yet supported in <notes>";
 999                         fireWarningEvent(message, next, 4);
1000                 }else{
1001                         String firstSubheading = stringMap.keySet().iterator().next();
1002                         if ( firstSubheading != null && ! isNoteHeading (firstSubheading) )  {
1003                                 String message = "Subheadings not yet supported in <notes>";
1004                                 fireWarningEvent(message, next, 4);
1005                         }
1006                 }
1007                 for (String subheading : stringMap.keySet()){
1008                         text += subheading;
1009                         text += stringMap.get(subheading);
1010                 }
1011                 return text;
1012         }
1013
1014         private boolean isNoteHeading(String heading) {
1015                 String excludePattern = "(i?)(Notes?):?";
1016                 return heading.matches(excludePattern);
1017         }
1018
1019         /**
1020          * @param state
1021          * @param attributes
1022          */
1023         private Taxon createTaxonAndName(MarkupImportState state,
1024                         Map<String, Attribute> attributes) {
1025                 NonViralName<?> name;
1026                 Rank rank = Rank.SPECIES(); // default
1027                 boolean isCultivar = checkAndRemoveAttributeValue(attributes, CLASS,
1028                                 "cultivated");
1029                 if (isCultivar) {
1030                         name = CultivarPlantName.NewInstance(rank);
1031                 } else {
1032                         name = createNameByCode(state, rank);
1033                 }
1034                 Taxon taxon = Taxon.NewInstance(name, state.getConfig()
1035                                 .getSourceReference());
1036                 if (checkAndRemoveAttributeValue(attributes, CLASS, "dubious")) {
1037                         taxon.setDoubtful(true);
1038                 } else if (checkAndRemoveAttributeValue(attributes, CLASS, "excluded")) {
1039                         taxon.setExcluded(true);
1040                 }
1041                 // TODO insufficient, new, expected
1042                 handleNotYetImplementedAttribute(attributes, CLASS);
1043                 // From old version
1044                 // MarkerType markerType = getMarkerType(state, attrValue);
1045                 // if (markerType == null){
1046                 // logger.warn("Class attribute value for taxon not yet supported: " +
1047                 // attrValue);
1048                 // }else{
1049                 // taxon.addMarker(Marker.NewInstance(markerType, true));
1050                 // }
1051
1052                 // save(name, state);
1053                 // save(taxon, state);
1054                 return taxon;
1055         }
1056
1057         /**
1058          * @param state
1059          * @param rank
1060          * @return
1061          */
1062         private NonViralName<?> createNameByCode(MarkupImportState state, Rank rank) {
1063                 NonViralName<?> name;
1064                 NomenclaturalCode nc = makeNomenclaturalCode(state);
1065                 name = (NonViralName<?>) nc.getNewTaxonNameInstance(rank);
1066                 return name;
1067         }
1068
1069         /**
1070          * @param state
1071          * @return
1072          */
1073         private NomenclaturalCode makeNomenclaturalCode(MarkupImportState state) {
1074                 NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
1075                 if (nc == null) {
1076                         nc = NomenclaturalCode.ICBN; // default;
1077                 }
1078                 return nc;
1079         }
1080
1081         private String handleTaxonTitle(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1082                 //attributes
1083                 String text = "";
1084                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1085                 String rankAttr = getAndRemoveAttributeValue(attributes, RANK);
1086                 Rank rank = makeRank(state, rankAttr, false);
1087                 String num = getAndRemoveAttributeValue(attributes, NUM);
1088                 state.setCurrentTaxonNum(num);
1089                 checkNoAttributes(attributes, parentEvent);
1090
1091                 // TODO handle attributes
1092                 while (reader.hasNext()) {
1093                         XMLEvent next = readNoWhitespace(reader);
1094                         if (next.isEndElement()) {
1095                                 if (isMyEndingElement(next, parentEvent)) {
1096                                         Taxon taxon = state.getCurrentTaxon();
1097                                         String titleText = null;
1098                                         if (checkMandatoryText(text, parentEvent)) {
1099                                                 titleText = normalize(text);
1100                                                 UUID uuidTitle = MarkupTransformer.uuidTaxonTitle;
1101                                                 ExtensionType titleExtension = this.getExtensionType(state, uuidTitle, "Taxon Title ","taxon title", "title");
1102                                                 taxon.addExtension(titleText, titleExtension);
1103                                         }
1104                                         taxon.getName().setRank(rank);
1105                                         // TODO check title exists
1106                                         return titleText;
1107                                 } else {
1108                                         if (isEndingElement(next, FOOTNOTE)) {
1109                                                 // NOT YET IMPLEMENTED
1110                                                 popUnimplemented(next.asEndElement());
1111                                         } else {
1112                                                 handleUnexpectedEndElement(next.asEndElement());
1113                                                 state.setUnsuccessfull();
1114                                         }
1115                                 }
1116                         } else if (next.isStartElement()) {
1117                                 if (isStartingElement(next, FOOTNOTE)) {
1118                                         handleNotYetImplementedElement(next);
1119                                 } else {
1120                                         handleUnexpectedStartElement(next);
1121                                         state.setUnsuccessfull();
1122                                 }
1123                         } else if (next.isCharacters()) {
1124                                 text += next.asCharacters().getData();
1125
1126                         } else {
1127                                 handleUnexpectedElement(next);
1128                                 state.setUnsuccessfull();
1129                         }
1130                 }
1131                 return null;
1132
1133         }
1134
1135         private WriterDataHolder handleWriter(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1136                 String text = "";
1137                 checkNoAttributes(parentEvent);
1138                 WriterDataHolder dataHolder = new WriterDataHolder();
1139                 List<FootnoteDataHolder> footnotes = new ArrayList<FootnoteDataHolder>();
1140
1141                 // TODO handle attributes
1142                 while (reader.hasNext()) {
1143                         XMLEvent next = readNoWhitespace(reader);
1144                         if (isMyEndingElement(next, parentEvent)) {
1145                                 text = CdmUtils.removeBrackets(text);
1146                                 if (checkMandatoryText(text, parentEvent)) {
1147                                         text = normalize(text);
1148                                         dataHolder.writer = text;
1149                                         dataHolder.footnotes = footnotes;
1150
1151                                         // Extension
1152                                         UUID uuidWriterExtension = MarkupTransformer.uuidWriterExtension;
1153                                         ExtensionType writerExtensionType = this
1154                                                         .getExtensionType(state, uuidWriterExtension,
1155                                                                         "Writer", "writer", "writer");
1156                                         Extension extension = Extension.NewInstance();
1157                                         extension.setType(writerExtensionType);
1158                                         extension.setValue(text);
1159                                         dataHolder.extension = extension;
1160
1161                                         // Annotation
1162                                         UUID uuidWriterAnnotation = MarkupTransformer.uuidWriterAnnotation;
1163                                         AnnotationType writerAnnotationType = this.getAnnotationType(state, uuidWriterAnnotation, "Writer", "writer", "writer", null);
1164                                         Annotation annotation = Annotation.NewInstance(text, writerAnnotationType, Language.DEFAULT());
1165                                         dataHolder.annotation = annotation;
1166
1167                                         return dataHolder;
1168                                 } else {
1169                                         return null;
1170                                 }
1171                         } else if (isStartingElement(next, FOOTNOTE_REF)) {
1172                                 FootnoteDataHolder footNote = handleFootnoteRef(state, reader, next);
1173                                 if (footNote.isRef()) {
1174                                         footnotes.add(footNote);
1175                                 } else {
1176                                         logger.warn("Non ref footnotes not yet impelemnted");
1177                                 }
1178                         } else if (next.isCharacters()) {
1179                                 text += next.asCharacters().getData();
1180
1181                         } else {
1182                                 handleUnexpectedElement(next);
1183                                 state.setUnsuccessfull();
1184                         }
1185                 }
1186                 throw new IllegalStateException("<writer> has no end tag");
1187         }
1188
1189         private void registerFootnotes(MarkupImportState state, AnnotatableEntity entity, List<FootnoteDataHolder> footnotes) {
1190                 for (FootnoteDataHolder footNote : footnotes) {
1191                         registerFootnoteDemand(state, entity, footNote);
1192                 }
1193         }
1194
1195         private void registerGivenFootnote(MarkupImportState state, FootnoteDataHolder footnote) {
1196                 state.registerFootnote(footnote);
1197                 Set<AnnotatableEntity> demands = state.getFootnoteDemands(footnote.id);
1198                 if (demands != null) {
1199                         for (AnnotatableEntity entity : demands) {
1200                                 attachFootnote(state, entity, footnote);
1201                         }
1202                 }
1203         }
1204
1205         private void registerGivenFigure(MarkupImportState state, String id, Media figure) {
1206                 state.registerFigure(id, figure);
1207                 Set<AnnotatableEntity> demands = state.getFigureDemands(id);
1208                 if (demands != null) {
1209                         for (AnnotatableEntity entity : demands) {
1210                                 attachFigure(state, entity, figure);
1211                         }
1212                 }
1213         }
1214
1215         private void registerFootnoteDemand(MarkupImportState state, AnnotatableEntity entity, FootnoteDataHolder footnote) {
1216                 FootnoteDataHolder existingFootnote = state.getFootnote(footnote.ref);
1217                 if (existingFootnote != null) {
1218                         attachFootnote(state, entity, existingFootnote);
1219                 } else {
1220                         Set<AnnotatableEntity> demands = state.getFootnoteDemands(footnote.ref);
1221                         if (demands == null) {
1222                                 demands = new HashSet<AnnotatableEntity>();
1223                                 state.putFootnoteDemands(footnote.ref, demands);
1224                         }
1225                         demands.add(entity);
1226                 }
1227         }
1228
1229         private void registerFigureDemand(MarkupImportState state, AnnotatableEntity entity, String figureRef) {
1230                 Media existingFigure = state.getFigure(figureRef);
1231                 if (existingFigure != null) {
1232                         attachFigure(state, entity, existingFigure);
1233                 } else {
1234                         Set<AnnotatableEntity> demands = state.getFigureDemands(figureRef);
1235                         if (demands == null) {
1236                                 demands = new HashSet<AnnotatableEntity>();
1237                                 state.putFigureDemands(figureRef, demands);
1238                         }
1239                         demands.add(entity);
1240                 }
1241         }
1242
1243         private void attachFootnote(MarkupImportState state, AnnotatableEntity entity, FootnoteDataHolder footnote) {
1244                 AnnotationType annotationType = this.getAnnotationType(state, MarkupTransformer.uuidFootnote, "Footnote", "An e-flora footnote", "fn", null);
1245                 Annotation annotation = Annotation.NewInstance(footnote.string,
1246                                 annotationType, Language.DEFAULT());
1247                 // TODO transient objects
1248                 entity.addAnnotation(annotation);
1249                 save(entity, state);
1250         }
1251
1252         private void attachFigure(MarkupImportState state,
1253                         AnnotatableEntity entity, Media figure) {
1254                 // IdentifiableEntity<?> toSave;
1255                 if (entity.isInstanceOf(TextData.class)) {
1256                         TextData deb = CdmBase.deproxy(entity, TextData.class);
1257                         deb.addMedia(figure);
1258                         // toSave = ((TaxonDescription)deb.getInDescription()).getTaxon();
1259                 } else if (entity.isInstanceOf(IdentifiableMediaEntity.class)) {
1260                         IdentifiableMediaEntity<?> ime = CdmBase.deproxy(entity,
1261                                         IdentifiableMediaEntity.class);
1262                         ime.addMedia(figure);
1263                         // toSave = ime;
1264                 } else {
1265                         String message = "Unsupported entity to attach media: %s";
1266                         message = String.format(message, entity.getClass().getName());
1267                         // toSave = null;
1268                 }
1269                 save(entity, state);
1270         }
1271
1272         private void handleFigure(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1273                 // FigureDataHolder result = new FigureDataHolder();
1274
1275                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1276                 String id = getAndRemoveAttributeValue(attributes, ID);
1277                 String type = getAndRemoveAttributeValue(attributes, TYPE);
1278                 checkNoAttributes(attributes, parentEvent);
1279
1280                 String urlString = null;
1281                 String legendString = null;
1282                 String titleString = null;
1283                 String numString = null;
1284                 String text = null;
1285                 while (reader.hasNext()) {
1286                         XMLEvent next = readNoWhitespace(reader);
1287                         if (isMyEndingElement(next, parentEvent)) {
1288                                 makeFigure(state, id, type, urlString, legendString, titleString, numString, next);
1289                                 return;
1290                         } else if (isStartingElement(next, FIGURE_LEGEND)) {
1291                                 // TODO same as figurestring ?
1292                                 legendString = handleFootnoteString(state, reader, next);
1293                         } else if (isStartingElement(next, FIGURE_TITLE)) {
1294                                 titleString = getCData(state, reader, next);
1295                         } else if (isStartingElement(next, URL)) {
1296                                 String localUrl = getCData(state, reader, next);
1297                                 urlString = CdmUtils.Nz(state.getBaseMediaUrl()) + localUrl;
1298                         } else if (isStartingElement(next, NUM)) {
1299                                 numString = getCData(state, reader, next);
1300                         } else if (next.isCharacters()) {
1301                                 text += next.asCharacters().getData();
1302                         } else {
1303                                 fireUnexpectedEvent(next, 0);
1304                         }
1305                 }
1306                 throw new IllegalStateException("<figure> has no end tag");
1307         }
1308
1309         /**
1310          * @param state
1311          * @param id
1312          * @param type
1313          * @param urlString
1314          * @param legendString
1315          * @param titleString
1316          * @param numString
1317          * @param next
1318          */
1319         private void makeFigure(MarkupImportState state, String id, String type, String urlString,
1320                                                 String legendString, String titleString, String numString, XMLEvent next) {
1321                 Media media = null;
1322                 boolean isFigure = false;
1323                 try {
1324                         //TODO maybe everything is a figure as it is all taken from a book
1325                         if ("lineart".equals(type)) {
1326                                 isFigure = true;
1327 //                              media = Figure.NewInstance(url.toURI(), null, null,     null);
1328                         } else if (type == null || "photo".equals(type)
1329                                         || "signature".equals(type)
1330                                         || "others".equals(type)) {
1331                         } else {
1332                                 String message = "Unknown figure type '%s'";
1333                                 message = String.format(message, type);
1334                                 fireWarningEvent(message, next, 2);
1335                         }
1336                         media = getImageMedia(urlString, READ_MEDIA_DATA, isFigure);
1337
1338                         if (media != null){
1339                                 // title
1340                                 if (StringUtils.isNotBlank(titleString)) {
1341                                         media.putTitle(Language.DEFAULT(), titleString);
1342                                 }
1343                                 // legend
1344                                 if (StringUtils.isNotBlank(legendString)) {
1345                                         media.addDescription(legendString, Language.DEFAULT());
1346                                 }
1347                                 if (StringUtils.isNotBlank(numString)) {
1348                                         // TODO use concrete source (e.g. DAPHNIPHYLLACEAE in FM
1349                                         // vol.13)
1350                                         Reference<?> citation = state.getConfig().getSourceReference();
1351                                         media.addSource(numString, "num", citation, null);
1352                                         // TODO name used in source if available
1353                                 }
1354                                 // TODO which citation
1355                                 if (StringUtils.isNotBlank(id)) {
1356                                         media.addSource(id, null, state.getConfig().getSourceReference(), null);
1357                                 } else {
1358                                         String message = "Figure id should never be empty or null";
1359                                         fireWarningEvent(message, next, 6);
1360                                 }
1361
1362                                 // text
1363                                 // do nothing
1364
1365                         }
1366                 } catch (MalformedURLException e) {
1367                         String message = "Media uri has incorrect syntax: %s";
1368                         message = String.format(message, urlString);
1369                         fireWarningEvent(message, next, 4);
1370 //              } catch (URISyntaxException e) {
1371 //                      String message = "Media uri has incorrect syntax: %s";
1372 //                      message = String.format(message, urlString);
1373 //                      fireWarningEvent(message, next, 4);
1374                 }
1375
1376                 registerGivenFigure(state, id, media);
1377         }
1378
1379         private FigureDataHolder handleFigureRef(MarkupImportState state,
1380                         XMLEventReader reader, XMLEvent parentEvent)
1381                         throws XMLStreamException {
1382                 FigureDataHolder result = new FigureDataHolder();
1383                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1384                 result.ref = getAndRemoveAttributeValue(attributes, REF);
1385                 checkNoAttributes(attributes, parentEvent);
1386
1387                 // text is not handled, needed only for debugging purposes
1388                 String text = "";
1389                 while (reader.hasNext()) {
1390                         XMLEvent next = readNoWhitespace(reader);
1391                         if (isMyEndingElement(next, parentEvent)) {
1392                                 return result;
1393                         } else if (isStartingElement(next, NUM)) {
1394                                 String num = getCData(state, reader, next);
1395                                 result.num = num; // num is not handled during import
1396                         } else if (isStartingElement(next, FIGURE_PART)) {
1397                                 result.figurePart = getCData(state, reader, next);
1398                         } else if (next.isCharacters()) {
1399                                 text += next.asCharacters().getData();
1400                         } else {
1401                                 fireUnexpectedEvent(next, 0);
1402                         }
1403                 }
1404                 throw new IllegalStateException("<figureRef> has no end tag");
1405         }
1406
1407         private FootnoteDataHolder handleFootnote(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1408                 FootnoteDataHolder result = new FootnoteDataHolder();
1409                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1410                 result.id = getAndRemoveAttributeValue(attributes, ID);
1411                 // result.ref = getAndRemoveAttributeValue(attributes, REF);
1412                 checkNoAttributes(attributes, parentEvent);
1413
1414                 while (reader.hasNext()) {
1415                         XMLEvent next = readNoWhitespace(reader);
1416                         if (isStartingElement(next, FOOTNOTE_STRING)) {
1417                                 String string = handleFootnoteString(state, reader, next);
1418                                 result.string = string;
1419                         } else if (isMyEndingElement(next, parentEvent)) {
1420                                 return result;
1421                         } else {
1422                                 fireUnexpectedEvent(next, 0);
1423                         }
1424                 }
1425                 return result;
1426         }
1427
1428         private FootnoteDataHolder handleFootnoteRef(MarkupImportState state,
1429                         XMLEventReader reader, XMLEvent parentEvent)
1430                         throws XMLStreamException {
1431                 FootnoteDataHolder result = new FootnoteDataHolder();
1432                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1433                 result.ref = getAndRemoveAttributeValue(attributes, REF);
1434                 checkNoAttributes(attributes, parentEvent);
1435
1436                 // text is not handled, needed only for debugging purposes
1437                 String text = "";
1438                 while (reader.hasNext()) {
1439                         XMLEvent next = readNoWhitespace(reader);
1440                         // if (isStartingElement(next, FOOTNOTE_STRING)){
1441                         // String string = handleFootnoteString(state, reader, next);
1442                         // result.string = string;
1443                         // }else
1444                         if (isMyEndingElement(next, parentEvent)) {
1445                                 return result;
1446                         } else if (next.isCharacters()) {
1447                                 text += next.asCharacters().getData();
1448
1449                         } else {
1450                                 fireUnexpectedEvent(next, 0);
1451                         }
1452                 }
1453                 return result;
1454         }
1455
1456         private void handleNomenclature(MarkupImportState state,
1457                         XMLEventReader reader, XMLEvent parentEvent)
1458                         throws XMLStreamException {
1459                 checkNoAttributes(parentEvent);
1460
1461                 while (reader.hasNext()) {
1462                         XMLEvent next = readNoWhitespace(reader);
1463                         if (isStartingElement(next, HOMOTYPES)) {
1464                                 handleHomotypes(state, reader, next.asStartElement());
1465                         } else if (isMyEndingElement(next, parentEvent)) {
1466                                 return;
1467                         } else {
1468                                 fireSchemaConflictEventExpectedStartTag(HOMOTYPES, reader);
1469                                 state.setUnsuccessfull();
1470                         }
1471                 }
1472                 return;
1473         }
1474
1475         private String handleFootnoteString(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1476                 boolean isTextMode = true;
1477                 String text = "";
1478                 while (reader.hasNext()) {
1479                         XMLEvent next = readNoWhitespace(reader);
1480                         if (isMyEndingElement(next, parentEvent)) {
1481                                 return text;
1482                         } else if (next.isEndElement()) {
1483                                 if (isEndingElement(next, FULL_NAME)) {
1484                                         popUnimplemented(next.asEndElement());
1485                                 } else if (isEndingElement(next, BR)) {
1486                                         isTextMode = true;
1487                                 } else if (isHtml(next)) {
1488                                         text += getXmlTag(next);
1489                                 } else {
1490                                         handleUnexpectedEndElement(next.asEndElement());
1491                                 }
1492                         } else if (next.isStartElement()) {
1493                                 if (isStartingElement(next, FULL_NAME)) {
1494                                         handleNotYetImplementedElement(next);
1495                                 } else if (isStartingElement(next, GATHERING)) {
1496                                         text += handleInLineGathering(state, reader, next);
1497                                 } else if (isStartingElement(next, REFERENCES)) {
1498                                         text += " " + handleInLineReferences(state, reader, next)+ " ";
1499                                 } else if (isStartingElement(next, BR)) {
1500                                         text += "<br/>";
1501                                         isTextMode = false;
1502                                 } else if (isHtml(next)) {
1503                                         text += getXmlTag(next);
1504                                 } else {
1505                                         handleUnexpectedStartElement(next.asStartElement());
1506                                 }
1507                         } else if (next.isCharacters()) {
1508                                 if (!isTextMode) {
1509                                         String message = "footnoteString is not in text mode";
1510                                         fireWarningEvent(message, next, 6);
1511                                 } else {
1512                                         text += next.asCharacters().getData().trim();
1513                                         // getCData(state, reader, next); does not work as we have inner tags like <references>
1514                                 }
1515                         } else {
1516                                 handleUnexpectedEndElement(next.asEndElement());
1517                         }
1518                 }
1519                 throw new IllegalStateException("<footnoteString> has no closing tag");
1520
1521         }
1522
1523         private String handleInLineGathering(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1524                 DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(DerivedUnitType.DerivedUnit.FieldObservation);
1525                 handleGathering(state, reader, parentEvent, null, facade);
1526                 FieldObservation fieldObservation = facade.innerFieldObservation();
1527                 String result = "<cdm:specimen uuid='%s'>%s</specimen>";
1528                 result = String.format(result, fieldObservation.getUuid(), fieldObservation.getTitleCache());
1529                 save(fieldObservation, state);
1530                 return result;
1531         }
1532
1533         private String handleInLineReferences(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1534                 checkNoAttributes(parentEvent);
1535
1536                 boolean hasReference = false;
1537                 String text = "";
1538                 while (reader.hasNext()) {
1539                         XMLEvent next = readNoWhitespace(reader);
1540                         if (isMyEndingElement(next, parentEvent)) {
1541                                 checkMandatoryElement(hasReference, parentEvent.asStartElement(), REFERENCE);
1542                                 return text;
1543                         } else if (isStartingElement(next, REFERENCE)) {
1544                                 text += handleInLineReference(state, reader, next);
1545                                 hasReference = true;
1546                         } else {
1547                                 handleUnexpectedElement(next);
1548                         }
1549                 }
1550                 throw new IllegalStateException("<References> has no closing tag");
1551         }
1552
1553         private String handleInLineReference(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent)throws XMLStreamException {
1554                 Reference<?> reference = handleReference(state, reader, parentEvent);
1555                 String result = "<cdm:ref uuid='%s'>%s</ref>";
1556                 result = String.format(result, reference.getUuid(), reference.getTitleCache());
1557                 save(reference, state);
1558                 return result;
1559         }
1560
1561         private Reference<?> handleReference(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent)throws XMLStreamException {
1562                 checkNoAttributes(parentEvent);
1563
1564                 boolean hasRefPart = false;
1565                 Map<String, String> refMap = new HashMap<String, String>();
1566                 while (reader.hasNext()) {
1567                         XMLEvent next = readNoWhitespace(reader);
1568                         if (isMyEndingElement(next, parentEvent)) {
1569                                 checkMandatoryElement(hasRefPart, parentEvent.asStartElement(),
1570                                                 REF_PART);
1571                                 Reference<?> reference = createReference(state, refMap, next);
1572                                 return reference;
1573                         } else if (isStartingElement(next, REF_PART)) {
1574                                 handleRefPart(state, reader, next, refMap);
1575                                 hasRefPart = true;
1576                         } else {
1577                                 handleUnexpectedElement(next);
1578                         }
1579                 }
1580                 // TODO handle missing end element
1581                 throw new IllegalStateException("<Reference> has no closing tag");
1582         }
1583
1584         private void handleHomotypes(MarkupImportState state,
1585                         XMLEventReader reader, StartElement parentEvent)
1586                         throws XMLStreamException {
1587                 checkNoAttributes(parentEvent);
1588
1589                 HomotypicalGroup homotypicalGroup = null;
1590
1591                 boolean hasNom = false;
1592                 while (reader.hasNext()) {
1593                         XMLEvent next = readNoWhitespace(reader);
1594                         if (next.isEndElement()) {
1595                                 if (isMyEndingElement(next, parentEvent)) {
1596                                         checkMandatoryElement(hasNom, parentEvent, NOM);
1597                                         return;
1598                                 } else {
1599                                         if (isEndingElement(next, NAME_TYPE)) {
1600                                                 state.setNameType(false);
1601                                         } else if (isEndingElement(next, NOTES)) {
1602                                                 // NOT YET IMPLEMENTED
1603                                                 popUnimplemented(next.asEndElement());
1604                                         } else {
1605                                                 handleUnexpectedEndElement(next.asEndElement());
1606                                         }
1607                                 }
1608                         } else if (next.isStartElement()) {
1609                                 if (isStartingElement(next, NOM)) {
1610                                         NonViralName<?> name = handleNom(state, reader, next,
1611                                                         homotypicalGroup);
1612                                         homotypicalGroup = name.getHomotypicalGroup();
1613                                         hasNom = true;
1614                                 } else if (isStartingElement(next, NAME_TYPE)) {
1615                                         state.setNameType(true);
1616                                         handleNameType(state, reader, next, homotypicalGroup);
1617                                 } else if (isStartingElement(next, SPECIMEN_TYPE)) {
1618                                         handleSpecimenType(state, reader, next, homotypicalGroup);
1619                                 } else if (isStartingElement(next, NOTES)) {
1620                                         handleNotYetImplementedElement(next);
1621                                 } else {
1622                                         handleUnexpectedStartElement(next);
1623                                 }
1624                         } else {
1625                                 handleUnexpectedElement(next);
1626                         }
1627                 }
1628                 // TODO handle missing end element
1629                 throw new IllegalStateException("Homotypes has no closing tag");
1630
1631         }
1632
1633         private void handleNameType(MarkupImportState state, XMLEventReader reader,
1634                         XMLEvent parentEvent, HomotypicalGroup homotypicalGroup)
1635                         throws XMLStreamException {
1636                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1637                 String typeStatus = getAndRemoveAttributeValue(attributes, TYPE_STATUS);
1638                 checkNoAttributes(attributes, parentEvent);
1639
1640                 NameTypeDesignationStatus status;
1641                 try {
1642                         status = NameTypeParser.parseNameTypeStatus(typeStatus);
1643                 } catch (UnknownCdmTypeException e) {
1644                         String message = "Type status could not be recognized: %s";
1645                         message = String.format(message, typeStatus);
1646                         fireWarningEvent(message, parentEvent, 4);
1647                         status = null;
1648                 }
1649
1650                 boolean hasNom = false;
1651                 while (reader.hasNext()) {
1652                         XMLEvent next = readNoWhitespace(reader);
1653                         if (next.isEndElement()) {
1654                                 if (isMyEndingElement(next, parentEvent)) {
1655                                         checkMandatoryElement(hasNom, parentEvent.asStartElement(),
1656                                                         NOM);
1657                                         state.setNameType(false);
1658                                         return;
1659                                 } else {
1660                                         if (isEndingElement(next, ACCEPTED_NAME)) {
1661                                                 // NOT YET IMPLEMENTED
1662                                                 popUnimplemented(next.asEndElement());
1663                                         } else {
1664                                                 handleUnexpectedEndElement(next.asEndElement());
1665                                         }
1666                                 }
1667                         } else if (next.isStartElement()) {
1668                                 if (isStartingElement(next, NOM)) {
1669                                         // TODO should we check if the type is always a species, is
1670                                         // this a rule?
1671                                         NonViralName<?> speciesName = handleNom(state, reader,
1672                                                         next, null);
1673                                         for (TaxonNameBase<?, ?> name : homotypicalGroup
1674                                                         .getTypifiedNames()) {
1675                                                 name.addNameTypeDesignation(speciesName, null, null,
1676                                                                 null, status, false, false, false, false);
1677                                         }
1678                                         hasNom = true;
1679                                 } else if (isStartingElement(next, ACCEPTED_NAME)) {
1680                                         handleNotYetImplementedElement(next);
1681                                 } else {
1682                                         handleUnexpectedStartElement(next);
1683                                 }
1684                         } else {
1685                                 handleUnexpectedElement(next);
1686                         }
1687                 }
1688                 // TODO handle missing end element
1689                 throw new IllegalStateException("Homotypes has no closing tag");
1690
1691         }
1692
1693         private void handleSpecimenType(MarkupImportState state,
1694                         XMLEventReader reader, XMLEvent parentEvent,
1695                         HomotypicalGroup homotypicalGroup) throws XMLStreamException {
1696                 // attributes
1697                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1698                 String typeStatus = getAndRemoveAttributeValue(attributes, TYPE_STATUS);
1699                 String notSeen = getAndRemoveAttributeValue(attributes, NOT_SEEN);
1700                 String unknown = getAndRemoveAttributeValue(attributes, UNKNOWN);
1701                 String notFound = getAndRemoveAttributeValue(attributes, NOT_FOUND);
1702                 String destroyed = getAndRemoveAttributeValue(attributes, DESTROYED);
1703                 String lost = getAndRemoveAttributeValue(attributes, LOST);
1704                 checkNoAttributes(attributes, parentEvent);
1705                 if (StringUtils.isNotEmpty(typeStatus)) {
1706                         // TODO
1707                         // currently not needed
1708                 } else if (StringUtils.isNotEmpty(notSeen)) {
1709                         handleNotYetImplementedAttribute(attributes, NOT_SEEN);
1710                 } else if (StringUtils.isNotEmpty(unknown)) {
1711                         handleNotYetImplementedAttribute(attributes, UNKNOWN);
1712                 } else if (StringUtils.isNotEmpty(notFound)) {
1713                         handleNotYetImplementedAttribute(attributes, NOT_FOUND);
1714                 } else if (StringUtils.isNotEmpty(destroyed)) {
1715                         handleNotYetImplementedAttribute(attributes, DESTROYED);
1716                 } else if (StringUtils.isNotEmpty(lost)) {
1717                         handleNotYetImplementedAttribute(attributes, LOST);
1718                 }
1719
1720                 NonViralName<?> firstName = null;
1721                 Set<TaxonNameBase> names = homotypicalGroup.getTypifiedNames();
1722                 if (names.isEmpty()) {
1723                         String message = "There is no name in a homotypical group. Can't create the specimen type";
1724                         fireWarningEvent(message, parentEvent, 8);
1725                 } else {
1726                         firstName = CdmBase.deproxy(names.iterator().next(),
1727                                         NonViralName.class);
1728                 }
1729
1730                 DerivedUnitFacade facade = DerivedUnitFacade
1731                                 .NewInstance(DerivedUnitType.Specimen);
1732                 String text = "";
1733                 // elements
1734                 while (reader.hasNext()) {
1735                         XMLEvent next = readNoWhitespace(reader);
1736                         if (next.isEndElement()) {
1737                                 if (isMyEndingElement(next, parentEvent)) {
1738                                         makeSpecimenType(state, facade, text, firstName,
1739                                                         parentEvent);
1740                                         return;
1741                                 } else {
1742                                         if (isEndingElement(next, FULL_TYPE)) {
1743                                                 // NOT YET IMPLEMENTED
1744                                                 popUnimplemented(next.asEndElement());
1745                                         } else if (isEndingElement(next, TYPE_STATUS)) {
1746                                                 // NOT YET IMPLEMENTED
1747                                                 popUnimplemented(next.asEndElement());
1748                                         } else if (isEndingElement(next, ORIGINAL_DETERMINATION)) {
1749                                                 // NOT YET IMPLEMENTED
1750                                                 popUnimplemented(next.asEndElement());
1751                                         } else if (isEndingElement(next, SPECIMEN_TYPE)) {
1752                                                 // NOT YET IMPLEMENTED
1753                                                 popUnimplemented(next.asEndElement());
1754                                         } else if (isEndingElement(next, COLLECTION_AND_TYPE)) {
1755                                                 // NOT YET IMPLEMENTED
1756                                                 popUnimplemented(next.asEndElement());
1757                                         } else if (isEndingElement(next, CITATION)) {
1758                                                 // NOT YET IMPLEMENTED
1759                                                 popUnimplemented(next.asEndElement());
1760                                         } else if (isEndingElement(next, NOTES)) {
1761                                                 // NOT YET IMPLEMENTED
1762                                                 popUnimplemented(next.asEndElement());
1763                                         } else if (isEndingElement(next, ANNOTATION)) {
1764                                                 // NOT YET IMPLEMENTED
1765                                                 popUnimplemented(next.asEndElement());
1766                                         } else {
1767                                                 handleUnexpectedEndElement(next.asEndElement());
1768                                         }
1769                                 }
1770                         } else if (next.isStartElement()) {
1771                                 if (isStartingElement(next, FULL_TYPE)) {
1772                                         handleNotYetImplementedElement(next);
1773                                         // homotypicalGroup = handleNom(state, reader, next, taxon,
1774                                         // homotypicalGroup);
1775                                 } else if (isStartingElement(next, TYPE_STATUS)) {
1776                                         handleNotYetImplementedElement(next);
1777                                 } else if (isStartingElement(next, GATHERING)) {
1778                                         handleGathering(state, reader, next, homotypicalGroup, facade);
1779                                 } else if (isStartingElement(next, ORIGINAL_DETERMINATION)) {
1780                                         handleNotYetImplementedElement(next);
1781                                 } else if (isStartingElement(next, SPECIMEN_TYPE)) {
1782                                         handleNotYetImplementedElement(next);
1783                                 } else if (isStartingElement(next, COLLECTION_AND_TYPE)) {
1784                                         handleNotYetImplementedElement(next);
1785                                 } else if (isStartingElement(next, NOTES)) {
1786                                         handleNotYetImplementedElement(next);
1787                                 } else if (isStartingElement(next, ANNOTATION)) {
1788                                         handleNotYetImplementedElement(next);
1789                                 } else {
1790                                         handleUnexpectedStartElement(next);
1791                                 }
1792                         } else if (next.isCharacters()) {
1793                                 text += next.asCharacters().getData();
1794                         } else {
1795                                 handleUnexpectedElement(next);
1796                         }
1797                 }
1798                 // TODO handle missing end element
1799                 throw new IllegalStateException("Specimen type has no closing tag");
1800         }
1801
1802         private void makeSpecimenType(MarkupImportState state,
1803                         DerivedUnitFacade facade, String text, NonViralName name,
1804                         XMLEvent parentEvent) {
1805                 text = text.trim();
1806                 // remove brackets
1807                 if (text.matches("^\\(.*\\)\\.?$")) {
1808                         text = text.replaceAll("\\.", "");
1809                         text = text.substring(1, text.length() - 1);
1810                 }
1811                 String[] split = text.split("[;,]");
1812                 for (String str : split) {
1813                         str = str.trim();
1814                         boolean addToAllNamesInGroup = true;
1815                         TypeInfo typeInfo = makeSpecimenTypeTypeInfo(str, parentEvent);
1816                         SpecimenTypeDesignationStatus typeStatus = typeInfo.status;
1817                         Collection collection = createCollection(typeInfo.collectionString);
1818
1819                         // TODO improve cache strategy handling
1820                         DerivedUnitBase typeSpecimen = facade.addDuplicate(collection,
1821                                         null, null, null, null);
1822                         typeSpecimen.setCacheStrategy(new DerivedUnitFacadeCacheStrategy());
1823                         name.addSpecimenTypeDesignation((Specimen) typeSpecimen, typeStatus, null, null, null, false, addToAllNamesInGroup);
1824                 }
1825         }
1826
1827         private Collection createCollection(String code) {
1828                 // TODO deduplicate
1829                 // TODO code <-> name
1830                 Collection result = Collection.NewInstance();
1831                 result.setCode(code);
1832                 return result;
1833         }
1834
1835         private TypeInfo makeSpecimenTypeTypeInfo(String originalString, XMLEvent event) {
1836                 TypeInfo result = new TypeInfo();
1837                 String[] split = originalString.split("\\s+");
1838                 for (String str : split) {
1839                         if (str.matches(SpecimenTypeParser.typeTypePattern)) {
1840                                 SpecimenTypeDesignationStatus status;
1841                                 try {
1842                                         status = SpecimenTypeParser.parseSpecimenTypeStatus(str);
1843                                 } catch (UnknownCdmTypeException e) {
1844                                         String message = "Specimen type status '%s' not recognized by parser";
1845                                         message = String.format(message, str);
1846                                         fireWarningEvent(message, event, 4);
1847                                         status = null;
1848                                 }
1849                                 result.status = status;
1850                         } else if (str.matches(SpecimenTypeParser.collectionPattern)) {
1851                                 result.collectionString = str;
1852                         } else {
1853                                 String message = "Type part '%s' could not be recognized";
1854                                 message = String.format(message, str);
1855                                 fireWarningEvent(message, event, 2);
1856                         }
1857                 }
1858
1859                 return result;
1860         }
1861
1862
1863         private void handleGathering(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent, HomotypicalGroup homotypicalGroup, DerivedUnitFacade facade) throws XMLStreamException {
1864                 checkNoAttributes(parentEvent);
1865                 boolean hasCollector = false;
1866                 boolean hasFieldNum = false;
1867
1868                 // elements
1869                 while (reader.hasNext()) {
1870                         XMLEvent next = readNoWhitespace(reader);
1871                         if (next.isEndElement()) {
1872                                 if (isMyEndingElement(next, parentEvent)) {
1873                                         checkMandatoryElement(hasCollector,parentEvent.asStartElement(), COLLECTOR);
1874                                         checkMandatoryElement(hasFieldNum,parentEvent.asStartElement(), FIELD_NUM);
1875                                         return;
1876                                 } else {
1877                                         if (isEndingElement(next, ALTERNATIVE_COLLECTOR)) {
1878                                                 // NOT YET IMPLEMENTED
1879                                                 popUnimplemented(next.asEndElement());
1880                                         } else if (isEndingElement(next, ALTERNATIVE_FIELD_NUM)) {
1881                                                 // NOT YET IMPLEMENTED
1882                                                 popUnimplemented(next.asEndElement());
1883                                         } else if (isEndingElement(next, COLLECTION_TYPE_STATUS)) {
1884                                                 // NOT YET IMPLEMENTED
1885                                                 popUnimplemented(next.asEndElement());
1886                                         } else if (isEndingElement(next,
1887                                                         ALTERNATIVE_COLLECTION_TYPE_STATUS)) {
1888                                                 // NOT YET IMPLEMENTED
1889                                                 popUnimplemented(next.asEndElement());
1890                                         } else if (isEndingElement(next, SUB_COLLECTION)) {
1891                                                 // NOT YET IMPLEMENTED
1892                                                 popUnimplemented(next.asEndElement());
1893                                         } else if (isEndingElement(next, DATES)) {
1894                                                 // NOT YET IMPLEMENTED
1895                                                 popUnimplemented(next.asEndElement());
1896                                         } else if (isEndingElement(next, NOTES)) {
1897                                                 // NOT YET IMPLEMENTED
1898                                                 popUnimplemented(next.asEndElement());
1899                                         } else {
1900                                                 handleUnexpectedEndElement(next.asEndElement());
1901                                         }
1902                                 }
1903                         } else if (next.isStartElement()) {
1904                                 if (isStartingElement(next, COLLECTOR)) {
1905                                         hasCollector = true;
1906                                         String collectorStr = getCData(state, reader, next);
1907                                         AgentBase<?> collector = createCollector(collectorStr);
1908                                         facade.setCollector(collector);
1909                                 } else if (isStartingElement(next, ALTERNATIVE_COLLECTOR)) {
1910                                         handleNotYetImplementedElement(next);
1911                                 } else if (isStartingElement(next, FIELD_NUM)) {
1912                                         hasFieldNum = true;
1913                                         String fieldNumStr = getCData(state, reader, next);
1914                                         facade.setFieldNumber(fieldNumStr);
1915                                 } else if (isStartingElement(next, ALTERNATIVE_FIELD_NUM)) {
1916                                         handleNotYetImplementedElement(next);
1917                                 } else if (isStartingElement(next, COLLECTION_TYPE_STATUS)) {
1918                                         handleNotYetImplementedElement(next);
1919                                 } else if (isStartingElement(next,
1920                                                 ALTERNATIVE_COLLECTION_TYPE_STATUS)) {
1921                                         handleNotYetImplementedElement(next);
1922                                 } else if (isStartingElement(next, SUB_COLLECTION)) {
1923                                         handleNotYetImplementedElement(next);
1924                                 } else if (isStartingElement(next, LOCALITY)) {
1925                                         handleLocality(state, reader, next, facade);
1926                                 } else if (isStartingElement(next, DATES)) {
1927                                         handleNotYetImplementedElement(next);
1928                                 } else if (isStartingElement(next, NOTES)) {
1929                                         handleNotYetImplementedElement(next);
1930                                 } else {
1931                                         handleUnexpectedStartElement(next);
1932                                 }
1933                         } else {
1934                                 handleUnexpectedElement(next);
1935                         }
1936                 }
1937                 // TODO handle missing end element
1938                 throw new IllegalStateException("Collection has no closing tag");
1939
1940         }
1941
1942         private void handleLocality(MarkupImportState state, XMLEventReader reader,XMLEvent parentEvent, DerivedUnitFacade facade)throws XMLStreamException {
1943                 String classValue = getClassOnlyAttribute(parentEvent);
1944                 boolean isLocality = false;
1945                 NamedAreaLevel areaLevel = null;
1946                 if ("locality".equalsIgnoreCase(classValue)) {
1947                         isLocality = true;
1948                 } else {
1949                         areaLevel = makeNamedAreaLevel(state, classValue, parentEvent);
1950                 }
1951
1952                 String text = "";
1953                 // elements
1954                 while (reader.hasNext()) {
1955                         XMLEvent next = readNoWhitespace(reader);
1956                         if (next.isEndElement()) {
1957                                 if (isMyEndingElement(next, parentEvent)) {
1958                                         if (StringUtils.isNotBlank(text)) {
1959                                                 text = normalize(text);
1960                                                 if (isLocality) {
1961                                                         facade.setLocality(text);
1962                                                 } else {
1963                                                         text = CdmUtils.removeTrailingDot(text);
1964                                                         NamedArea area = makeArea(state, text, areaLevel);
1965                                                         facade.addCollectingArea(area);
1966                                                 }
1967                                         }
1968                                         // TODO
1969                                         return;
1970                                 } else {
1971                                         if (isEndingElement(next, ALTITUDE)) {
1972                                                 // NOT YET IMPLEMENTED
1973                                                 popUnimplemented(next.asEndElement());
1974                                         } else if (isEndingElement(next, COORDINATES)) {
1975                                                 // NOT YET IMPLEMENTED
1976                                                 popUnimplemented(next.asEndElement());
1977                                         } else if (isEndingElement(next, ANNOTATION)) {
1978                                                 // NOT YET IMPLEMENTED
1979                                                 popUnimplemented(next.asEndElement());
1980                                         } else {
1981                                                 handleUnexpectedEndElement(next.asEndElement());
1982                                         }
1983                                 }
1984                         } else if (next.isStartElement()) {
1985                                 if (isStartingElement(next, ALTITUDE)) {
1986                                         handleNotYetImplementedElement(next);
1987                                         // homotypicalGroup = handleNom(state, reader, next, taxon,
1988                                         // homotypicalGroup);
1989                                 } else if (isStartingElement(next, COORDINATES)) {
1990                                         handleNotYetImplementedElement(next);
1991                                 } else if (isStartingElement(next, ANNOTATION)) {
1992                                         handleNotYetImplementedElement(next);
1993                                 } else {
1994                                         handleUnexpectedStartElement(next);
1995                                 }
1996                         } else if (next.isCharacters()) {
1997                                 text += next.asCharacters().getData();
1998                         } else {
1999                                 handleUnexpectedElement(next);
2000                         }
2001                 }
2002                 throw new IllegalStateException("<SpecimenType> has no closing tag");
2003         }
2004
2005 //      private NamedArea createArea(String text, NamedAreaLevel areaLevel, MarkupImportState state) {
2006 //              NamedArea area = NamedArea.NewInstance(text, text, null);
2007 //              area.setLevel(areaLevel);
2008 //              save(area, state);
2009 //              return area;
2010 //      }
2011
2012         private AgentBase<?> createCollector(String collectorStr) {
2013                 return createAuthor(collectorStr);
2014         }
2015
2016         private String getCData(MarkupImportState state, XMLEventReader reader, XMLEvent next) throws XMLStreamException {
2017                 return getCData(state, reader, next, true);
2018         }
2019
2020         /**
2021          * Reads character data. Any element other than character data or the ending
2022          * tag will fire an unexpected element event.
2023          *
2024          * @param state
2025          * @param reader
2026          * @param next
2027          * @return
2028          * @throws XMLStreamException
2029          */
2030         private String getCData(MarkupImportState state, XMLEventReader reader, XMLEvent next,boolean checkAttributes) throws XMLStreamException {
2031                 if (checkAttributes){
2032                         checkNoAttributes(next);
2033                 }
2034
2035                 String text = "";
2036                 while (reader.hasNext()) {
2037                         XMLEvent myNext = readNoWhitespace(reader);
2038                         if (isMyEndingElement(myNext, next)) {
2039                                 return text;
2040                         } else if (myNext.isCharacters()) {
2041                                 text += myNext.asCharacters().getData();
2042                         } else {
2043                                 handleUnexpectedElement(myNext);
2044                         }
2045                 }
2046                 throw new IllegalStateException("Event has no closing tag");
2047
2048         }
2049
2050         /**
2051          * Creates the name defined by a nom tag. Adds it to the given homotypical
2052          * group (if not null).
2053          *
2054          * @param state
2055          * @param reader
2056          * @param parentEvent
2057          * @param homotypicalGroup
2058          * @return
2059          * @throws XMLStreamException
2060          */
2061         private NonViralName<?> handleNom(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent,
2062                         HomotypicalGroup homotypicalGroup) throws XMLStreamException {
2063                 boolean isSynonym = false;
2064                 boolean isNameType = state.isNameType();
2065                 // attributes
2066                 String classValue = getClassOnlyAttribute(parentEvent);
2067                 NonViralName<?> name;
2068                 if (!isNameType && ACCEPTED.equalsIgnoreCase(classValue)) {
2069                         isSynonym = false;
2070                         name = createName(state, homotypicalGroup, isSynonym);
2071                 } else if (!isNameType && SYNONYM.equalsIgnoreCase(classValue)) {
2072                         isSynonym = true;
2073                         name = createName(state, homotypicalGroup, isSynonym);
2074                 } else if (isNameType && NAME_TYPE.equalsIgnoreCase(classValue)) {
2075                         // TODO do we need to define the rank here?
2076                         name = createNameByCode(state, null);
2077                 } else {
2078                         fireUnexpectedAttributeValue(parentEvent, CLASS, classValue);
2079                         name = createNameByCode(state, null);
2080                 }
2081
2082                 Map<String, String> nameMap = new HashMap<String, String>();
2083
2084                 while (reader.hasNext()) {
2085                         XMLEvent next = readNoWhitespace(reader);
2086                         if (next.isEndElement()) {
2087                                 if (isMyEndingElement(next, parentEvent)) {
2088                                         // fill the name with all data gathered
2089                                         fillName(state, nameMap, name, next);
2090                                         return name;
2091                                 } else {
2092                                         if (isEndingElement(next, FULL_NAME)) {
2093                                                 // NOT YET IMPLEMENTED
2094                                                 popUnimplemented(next.asEndElement());
2095                                         } else if (isEndingElement(next, NUM)) {
2096                                                 // NOT YET IMPLEMENTED
2097                                                 popUnimplemented(next.asEndElement());
2098                                         } else if (isEndingElement(next, HOMONYM)) {
2099                                                 // NOT YET IMPLEMENTED
2100                                                 popUnimplemented(next.asEndElement());
2101                                         } else if (isEndingElement(next, NOTES)) {
2102                                                 // NOT YET IMPLEMENTED
2103                                                 popUnimplemented(next.asEndElement());
2104                                         } else if (isEndingElement(next, ANNOTATION)) {
2105                                                 // NOT YET IMPLEMENTED
2106                                                 popUnimplemented(next.asEndElement());
2107                                         } else {
2108                                                 handleUnexpectedEndElement(next.asEndElement());
2109                                         }
2110                                 }
2111                         } else if (next.isStartElement()) {
2112                                 if (isStartingElement(next, FULL_NAME)) {
2113                                         handleNotYetImplementedElement(next);
2114                                         // homotypicalGroup = handleNom(state, reader, next, taxon,
2115                                         // homotypicalGroup);
2116                                 } else if (isStartingElement(next, NUM)) {
2117                                         handleNotYetImplementedElement(next);
2118                                 } else if (isStartingElement(next, NAME)) {
2119                                         handleName(state, reader, next, nameMap);
2120                                 } else if (isStartingElement(next, CITATION)) {
2121                                         handleCitation(state, reader, next, name);
2122                                 } else if (isStartingElement(next, HOMONYM)) {
2123                                         handleNotYetImplementedElement(next);
2124                                 } else if (isStartingElement(next, NOTES)) {
2125                                         handleNotYetImplementedElement(next);
2126                                 } else if (isStartingElement(next, ANNOTATION)) {
2127                                         handleNotYetImplementedElement(next);
2128                                 } else {
2129                                         handleUnexpectedStartElement(next);
2130                                 }
2131                         } else {
2132                                 handleUnexpectedElement(next);
2133                         }
2134                 }
2135                 // TODO handle missing end element
2136                 throw new IllegalStateException("Nom has no closing tag");
2137
2138         }
2139
2140         private void fillName(MarkupImportState state, Map<String, String> nameMap,
2141                         NonViralName name, XMLEvent event) {
2142
2143                 // Ranks: family, subfamily, tribus, genus, subgenus, section,
2144                 // subsection, species, subspecies, variety, subvariety, forma
2145                 // infrank, paraut, author, infrparaut, infraut, status, notes
2146
2147                 String infrank = getAndRemoveMapKey(nameMap, INFRANK);
2148                 String authorStr = getAndRemoveMapKey(nameMap, AUTHOR);
2149                 String paraut = getAndRemoveMapKey(nameMap, PARAUT);
2150
2151                 String infrParAut = getAndRemoveMapKey(nameMap, INFRPARAUT);
2152                 String infrAut = getAndRemoveMapKey(nameMap, INFRAUT);
2153
2154                 String statusStr = getAndRemoveMapKey(nameMap, STATUS);
2155                 String notes = getAndRemoveMapKey(nameMap, NOTES);
2156
2157                 makeRankDecision(state, nameMap, name, event, infrank);
2158
2159                 // test consistency of rank and authors
2160                 testRankAuthorConsistency(name, event, authorStr, paraut, infrParAut,infrAut);
2161
2162                 // authors
2163                 makeNomenclaturalAuthors(name, event, authorStr, paraut, infrParAut,infrAut);
2164
2165                 // status
2166                 // TODO handle pro parte, pro syn. etc.
2167                 if (StringUtils.isNotBlank(statusStr)) {
2168                         String proPartePattern = "(pro parte|p.p.)";
2169                         if (statusStr.matches(proPartePattern)) {
2170                                 state.setProParte(true);
2171                         }
2172                         try {
2173                                 // TODO handle trim earlier
2174                                 statusStr = statusStr.trim();
2175                                 NomenclaturalStatusType nomStatusType = NomenclaturalStatusType
2176                                                 .getNomenclaturalStatusTypeByAbbreviation(statusStr);
2177                                 name.addStatus(NomenclaturalStatus.NewInstance(nomStatusType));
2178                         } catch (UnknownCdmTypeException e) {
2179                                 String message = "Status '%s' could not be recognized";
2180                                 message = String.format(message, statusStr);
2181                                 fireWarningEvent(message, event, 4);
2182                         }
2183                 }
2184
2185                 // notes
2186                 if (StringUtils.isNotBlank(notes)) {
2187                         handleNotYetImplementedAttributeValue(event, CLASS, NOTES);
2188                 }
2189
2190                 return;
2191         }
2192
2193         /**
2194          * @param state
2195          * @param nameMap
2196          * @param name
2197          * @param event
2198          * @param infrankStr
2199          */
2200         private void makeRankDecision(MarkupImportState state,
2201                         Map<String, String> nameMap, NonViralName<?> name, XMLEvent event,
2202                         String infrankStr) {
2203                 // TODO ranks
2204                 for (String key : nameMap.keySet()) {
2205                         Rank rank = makeRank(state, key, false);
2206                         if (rank == null) {
2207                                 handleNotYetImplementedAttributeValue(event, CLASS, key);
2208                         } else {
2209                                 if (name.getRank() == null || rank.isLower(name.getRank())) {
2210                                         name.setRank(rank);
2211                                 }
2212                                 String value = nameMap.get(key);
2213                                 if (rank.isSupraGeneric() || rank.isGenus()) {
2214                                         name.setGenusOrUninomial(value);
2215                                 } else if (rank.isInfraGeneric()) {
2216                                         name.setInfraGenericEpithet(value);
2217                                 } else if (rank.isSpecies()) {
2218                                         name.setSpecificEpithet(value);
2219                                 } else if (rank.isInfraSpecific()) {
2220                                         name.setInfraSpecificEpithet(value);
2221                                 } else {
2222                                         String message = "Invalid rank '%s'. Can't decide which epithet to fill with '%s'";
2223                                         message = String.format(message, rank.getTitleCache(),value);
2224                                         fireWarningEvent(message, event, 4);
2225                                 }
2226                         }
2227
2228                 }
2229                 // handle given infrank marker
2230                 if (StringUtils.isNotBlank(infrankStr)) {
2231                         Rank infRank = makeRank(state, infrankStr, true);
2232
2233                         if (infRank == null) {
2234                                 String message = "Infrank '%s' rank not recognized";
2235                                 message = String.format(message, infrankStr);
2236                                 fireWarningEvent(message, event, 4);
2237                         } else {
2238                                 if (name.getRank() == null) {
2239                                         name.setRank(infRank);
2240                                 } else if (infRank.isLower(name.getRank())) {
2241                                         String message = "InfRank '%s' is lower than existing rank ";
2242                                         message = String.format(message, infrankStr);
2243                                         fireWarningEvent(message, event, 2);
2244                                         name.setRank(infRank);
2245                                 } else if (infRank.equals(name.getRank())) {
2246                                         // nothing
2247                                 } else {
2248                                         String message = "InfRank '%s' is higher than existing rank ";
2249                                         message = String.format(message, infrankStr);
2250                                         fireWarningEvent(message, event, 2);
2251                                 }
2252                         }
2253                 }
2254         }
2255
2256         /**
2257          * @param name
2258          * @param event
2259          * @param authorStr
2260          * @param paraut
2261          * @param infrParAut
2262          * @param infrAut
2263          */
2264         private void makeNomenclaturalAuthors(NonViralName name, XMLEvent event,
2265                                 String authorStr, String paraut, String infrParAut, String infrAut) {
2266                 if (name.getRank() != null && name.getRank().isInfraSpecific()) {
2267                         if (StringUtils.isNotBlank(infrAut)) {
2268                                 INomenclaturalAuthor[] authorAndEx = authorAndEx(infrAut, event);
2269                                 name.setCombinationAuthorTeam(authorAndEx[0]);
2270                                 name.setExCombinationAuthorTeam(authorAndEx[1]);
2271                         }
2272                         if (StringUtils.isNotBlank(infrParAut)) {
2273                                 INomenclaturalAuthor[] authorAndEx = authorAndEx(infrParAut, event);
2274                                 name.setBasionymAuthorTeam(authorAndEx[0]);
2275                                 name.setExBasionymAuthorTeam(authorAndEx[1]);
2276                         }
2277                 } else {
2278                         if (name.getRank() == null){
2279                                 String message = "No rank defined. Check correct usage of authors!";
2280                                 fireWarningEvent(message, event, 4);
2281                                 if (isNotBlank(infrParAut) || isNotBlank(infrAut)){
2282                                         authorStr = infrAut;
2283                                         paraut = infrParAut;
2284                                 }
2285                         }
2286                         if (StringUtils.isNotBlank(authorStr)) {
2287                                 INomenclaturalAuthor[] authorAndEx = authorAndEx(authorStr, event);
2288                                 name.setCombinationAuthorTeam(authorAndEx[0]);
2289                                 name.setExCombinationAuthorTeam(authorAndEx[1]);
2290                         }
2291                         if (StringUtils.isNotBlank(paraut)) {
2292                                 INomenclaturalAuthor[] authorAndEx = authorAndEx(paraut, event);
2293                                 name.setBasionymAuthorTeam(authorAndEx[0]);
2294                                 name.setExBasionymAuthorTeam(authorAndEx[1]);
2295                         }
2296                 }
2297         }
2298
2299         private TeamOrPersonBase[] authorAndEx(String authorAndEx, XMLEvent xmlEvent) {
2300                 authorAndEx = authorAndEx.trim();
2301                 TeamOrPersonBase[] result = new TeamOrPersonBase[2];
2302
2303                 String[] split = authorAndEx.split("\\sex\\s");
2304                 if (split.length > 2) {
2305                         String message = "There is more then 1 ' ex ' in author string. Can't separate author and ex-author";
2306                         fireWarningEvent(message, xmlEvent, 4);
2307                         result[0] = createAuthor(authorAndEx);
2308                 } else if (split.length == 2) {
2309                         result[0] = createAuthor(split[1]);
2310                         result[1] = createAuthor(split[0]);
2311                 } else {
2312                         result[0] = createAuthor(split[0]);
2313                 }
2314                 return result;
2315         }
2316
2317         /**
2318          * Tests if the names rank is consistent with the given author strings.
2319          * @param name
2320          * @param event
2321          * @param authorStr
2322          * @param paraut
2323          * @param infrParAut
2324          * @param infrAut
2325          */
2326         private void testRankAuthorConsistency(NonViralName name, XMLEvent event,
2327                                         String authorStr, String paraut, String infrParAut, String infrAut) {
2328                 if (name.getRank() == null){
2329                         return;
2330                 }
2331                 if (name.getRank().isInfraSpecific()) {
2332                         if (StringUtils.isBlank(infrParAut)
2333                                         && StringUtils.isNotBlank(infrAut)
2334                                         && (StringUtils.isNotBlank(paraut) || StringUtils.isNotBlank(authorStr))) {
2335                                 String message = "Rank is infraspecicific but has only specific or higher author(s)";
2336                                 fireWarningEvent(message, event, 4);
2337                         }
2338                 } else {
2339                         // is not infraspecific
2340                         if (StringUtils.isNotBlank(infrParAut)  || StringUtils.isNotBlank(infrAut)) {
2341                                 String message = "Rank is not infraspecicific but name has infra author(s)";
2342                                 fireWarningEvent(message, event, 4);
2343                         }
2344                 }
2345         }
2346
2347         /**
2348          * Returns the (empty) name with the correct homotypical group depending on
2349          * the taxon status. Throws NPE if no currentTaxon is set in state.
2350          *
2351          * @param state
2352          * @param homotypicalGroup
2353          * @param isSynonym
2354          * @return
2355          */
2356         private NonViralName<?> createName(MarkupImportState state,
2357                         HomotypicalGroup homotypicalGroup, boolean isSynonym) {
2358                 NonViralName<?> name;
2359                 Taxon taxon = state.getCurrentTaxon();
2360                 if (isSynonym) {
2361                         Rank defaultRank = Rank.SPECIES(); // can be any
2362                         name = createNameByCode(state, defaultRank);
2363                         if (homotypicalGroup != null) {
2364                                 name.setHomotypicalGroup(homotypicalGroup);
2365                         }
2366                         SynonymRelationshipType synonymType = SynonymRelationshipType
2367                                         .HETEROTYPIC_SYNONYM_OF();
2368                         if (taxon.getHomotypicGroup().equals(homotypicalGroup)) {
2369                                 synonymType = SynonymRelationshipType.HOMOTYPIC_SYNONYM_OF();
2370                         }
2371                         taxon.addSynonymName(name, synonymType);
2372                 } else {
2373                         name = CdmBase.deproxy(taxon.getName(), NonViralName.class);
2374                 }
2375                 return name;
2376         }
2377
2378         private void handleName(MarkupImportState state, XMLEventReader reader,
2379                         XMLEvent parentEvent, Map<String, String> nameMap)
2380                         throws XMLStreamException {
2381                 String classValue = getClassOnlyAttribute(parentEvent);
2382
2383                 String text = "";
2384                 while (reader.hasNext()) {
2385                         XMLEvent next = readNoWhitespace(reader);
2386                         if (isMyEndingElement(next, parentEvent)) {
2387                                 nameMap.put(classValue, text);
2388                                 return;
2389                         } else if (next.isStartElement()) {
2390                                 if (isStartingElement(next, ANNOTATION)) {
2391                                         handleNotYetImplementedElement(next);
2392                                 } else {
2393                                         handleUnexpectedStartElement(next.asStartElement());
2394                                 }
2395                         } else if (next.isCharacters()) {
2396                                 text += next.asCharacters().getData();
2397                         } else {
2398                                 handleUnexpectedEndElement(next.asEndElement());
2399                         }
2400                 }
2401                 throw new IllegalStateException("name has no closing tag");
2402
2403         }
2404
2405         /**
2406          * @param state
2407          * @param classValue
2408          * @param byAbbrev
2409          * @return
2410          */
2411         private Rank makeRank(MarkupImportState state, String value,
2412                         boolean byAbbrev) {
2413                 Rank rank = null;
2414                 if (StringUtils.isBlank(value)) {
2415                         return null;
2416                 }
2417                 try {
2418                         boolean useUnknown = true;
2419                         NomenclaturalCode nc = makeNomenclaturalCode(state);
2420                         if (byAbbrev) {
2421                                 rank = Rank.getRankByAbbreviation(value, nc, useUnknown);
2422                         } else {
2423                                 rank = Rank.getRankByEnglishName(value, nc, useUnknown);
2424                         }
2425                         if (rank.equals(Rank.UNKNOWN_RANK())) {
2426                                 rank = null;
2427                         }
2428                 } catch (UnknownCdmTypeException e) {
2429                         // doNothing
2430                 }
2431                 return rank;
2432         }
2433
2434         // public void handleNameNotRank(MarkupImportState state, XMLEventReader
2435         // reader, XMLEvent parentEvent, String classValue, NonViralName name)
2436         // throws XMLStreamException {
2437         // if (ACCEPTED.equalsIgnoreCase(classValue)){
2438         // }else if (SYNONYM.equalsIgnoreCase(classValue)){
2439         // }else{
2440         // //TODO Not yet implemented
2441         // handleNotYetImplementedAttributeValue(parentEvent, CLASS, classValue);
2442         // }
2443         // }
2444
2445         private void handleCitation(MarkupImportState state, XMLEventReader reader,     XMLEvent parentEvent, NonViralName name) throws XMLStreamException {
2446                 String classValue = getClassOnlyAttribute(parentEvent);
2447
2448                 state.setCitation(true);
2449                 boolean hasRefPart = false;
2450                 Map<String, String> refMap = new HashMap<String, String>();
2451                 while (reader.hasNext()) {
2452                         XMLEvent next = readNoWhitespace(reader);
2453                         if (isMyEndingElement(next, parentEvent)) {
2454                                 checkMandatoryElement(hasRefPart, parentEvent.asStartElement(),
2455                                                 REF_PART);
2456                                 Reference<?> reference = createReference(state, refMap, next);
2457                                 String microReference = refMap.get(DETAILS);
2458                                 doCitation(state, name, classValue, reference, microReference,
2459                                                 parentEvent);
2460                                 state.setCitation(false);
2461                                 return;
2462                         } else if (isStartingElement(next, REF_PART)) {
2463                                 handleRefPart(state, reader, next, refMap);
2464                                 hasRefPart = true;
2465                         } else {
2466                                 handleUnexpectedElement(next);
2467                         }
2468                 }
2469                 throw new IllegalStateException("Citation has no closing tag");
2470
2471         }
2472
2473         private void handleRefPart(MarkupImportState state, XMLEventReader reader,XMLEvent parentEvent, Map<String, String> refMap) throws XMLStreamException {
2474                 String classValue = getClassOnlyAttribute(parentEvent);
2475
2476                 String text = "";
2477                 while (reader.hasNext()) {
2478                         XMLEvent next = readNoWhitespace(reader);
2479                         if (isMyEndingElement(next, parentEvent)) {
2480                                 refMap.put(classValue, text);
2481                                 return;
2482                         } else if (next.isStartElement()) {
2483                                 if (isStartingElement(next, ANNOTATION)) {
2484                                         handleNotYetImplementedElement(next);
2485                                 } else if (isStartingElement(next, ITALICS)) {
2486                                         handleNotYetImplementedElement(next);
2487                                 } else if (isStartingElement(next, BOLD)) {
2488                                         handleNotYetImplementedElement(next);
2489                                 } else {
2490                                         handleUnexpectedStartElement(next.asStartElement());
2491                                 }
2492                         } else if (next.isCharacters()) {
2493                                 text += next.asCharacters().getData();
2494                         } else {
2495                                 handleUnexpectedEndElement(next.asEndElement());
2496                         }
2497                 }
2498                 throw new IllegalStateException("RefPart has no closing tag");
2499
2500         }
2501
2502         private Reference<?> createReference(MarkupImportState state, Map<String, String> refMap, XMLEvent parentEvent) {
2503                 // TODO
2504                 Reference<?> reference;
2505
2506                 String type = getAndRemoveMapKey(refMap, PUBTYPE);
2507                 String authorStr = getAndRemoveMapKey(refMap, AUTHOR);
2508                 String titleStr = getAndRemoveMapKey(refMap, PUBTITLE);
2509                 String titleCache = getAndRemoveMapKey(refMap, PUBFULLNAME);
2510                 String volume = getAndRemoveMapKey(refMap, VOLUME);
2511                 String edition = getAndRemoveMapKey(refMap, EDITION);
2512                 String editors = getAndRemoveMapKey(refMap, EDITORS);
2513                 String year = getAndRemoveMapKey(refMap, YEAR);
2514                 String pubName = getAndRemoveMapKey(refMap, PUBNAME);
2515                 String pages = getAndRemoveMapKey(refMap, PAGES);
2516
2517                 if (state.isCitation()) {
2518                         if (volume != null || "journal".equalsIgnoreCase(type)) {
2519                                 IArticle article = ReferenceFactory.newArticle();
2520                                 if (pubName != null) {
2521                                         IJournal journal = ReferenceFactory.newJournal();
2522                                         journal.setTitle(pubName);
2523                                         article.setInJournal(journal);
2524                                 }
2525                                 reference = (Reference<?>) article;
2526
2527                         } else {
2528                                 // TODO
2529                                 if (pubName != null){
2530                                         reference  = ReferenceFactory.newBookSection();
2531                                 }else{
2532                                         reference = ReferenceFactory.newBook();
2533                                 }
2534                         }
2535                         // TODO use existing author from name or before
2536                         TeamOrPersonBase<?> author = createAuthor(authorStr);
2537                         reference.setAuthorTeam(author);
2538
2539                         reference.setTitle(titleStr);
2540                         if (StringUtils.isNotBlank(titleCache)) {
2541                                 reference.setTitleCache(titleCache, true);
2542                         }
2543                         reference.setEdition(edition);
2544                         reference.setEditor(editors);
2545
2546                         if (pubName != null) {
2547                                 Reference<?> inReference;
2548                                 if (reference.getType().equals(ReferenceType.Article)) {
2549                                         inReference = ReferenceFactory.newJournal();
2550                                 } else {
2551                                         inReference = ReferenceFactory.newGeneric();
2552                                 }
2553                                 inReference.setTitle(pubName);
2554                                 reference.setInReference(inReference);
2555                         }
2556
2557
2558                 } else {  //no citation
2559                         if (volume != null || "journal".equalsIgnoreCase(type)) {
2560                                 IArticle article = ReferenceFactory.newArticle();
2561                                 if (pubName != null) {
2562                                         IJournal journal = ReferenceFactory.newJournal();
2563                                         journal.setTitle(pubName);
2564                                         article.setInJournal(journal);
2565                                 }
2566                                 reference = (Reference<?>) article;
2567
2568                         } else {
2569                                 Reference<?> bookOrPartOf = ReferenceFactory.newGeneric();
2570                                 reference = bookOrPartOf;
2571                         }
2572
2573                         // TODO type
2574                         TeamOrPersonBase<?> author = createAuthor(authorStr);
2575                         reference.setAuthorTeam(author);
2576
2577                         reference.setTitle(titleStr);
2578                         if (StringUtils.isNotBlank(titleCache)) {
2579                                 reference.setTitleCache(titleCache, true);
2580                         }
2581                         reference.setEdition(edition);
2582                         reference.setEditor(editors);
2583
2584                         if (pubName != null) {
2585                                 Reference<?> inReference;
2586                                 if (reference.getType().equals(ReferenceType.Article)) {
2587                                         inReference = ReferenceFactory.newJournal();
2588                                 } else {
2589                                         inReference = ReferenceFactory.newGeneric();
2590                                 }
2591                                 inReference.setTitle(pubName);
2592                                 reference.setInReference(inReference);
2593                         }
2594                 }
2595                 reference.setVolume(volume);
2596                 reference.setDatePublished(TimePeriod.parseString(year));
2597                 //TODO check if this is handled correctly in FM markup
2598                 reference.setPages(pages);
2599
2600                 // TODO
2601                 String[] unhandledList = new String[]{ALTERNATEPUBTITLE, ISSUE, NOTES, STATUS};
2602                 for (String unhandled : unhandledList){
2603                         String value = getAndRemoveMapKey(refMap, unhandled);
2604                         if (isNotBlank(value)){
2605                                 this.handleNotYetImplementedAttributeValue(parentEvent, CLASS, unhandled);
2606                         }
2607                 }
2608
2609                 for (String key : refMap.keySet()) {
2610                         if (!DETAILS.equalsIgnoreCase(key)) {
2611                                 this.fireUnexpectedAttributeValue(parentEvent, CLASS, key);
2612                         }
2613                 }
2614
2615                 return reference;
2616         }
2617
2618         private TeamOrPersonBase createAuthor(String authorTitle) {
2619                 // TODO atomize and also use by name creation
2620                 TeamOrPersonBase result = Team.NewTitledInstance(authorTitle,
2621                                 authorTitle);
2622                 return result;
2623         }
2624
2625         private String getAndRemoveMapKey(Map<String, String> map, String key) {
2626                 String result = map.get(key);
2627                 map.remove(key);
2628                 if (result != null) {
2629                         result = normalize(result);
2630                 }
2631                 return StringUtils.stripToNull(result);
2632         }
2633
2634         private void doCitation(MarkupImportState state, NonViralName name,
2635                         String classValue, Reference reference, String microCitation,
2636                         XMLEvent parentEvent) {
2637                 if (PUBLICATION.equalsIgnoreCase(classValue)) {
2638                         name.setNomenclaturalReference(reference);
2639                         name.setNomenclaturalMicroReference(microCitation);
2640                 } else if (USAGE.equalsIgnoreCase(classValue)) {
2641                         Taxon taxon = state.getCurrentTaxon();
2642                         TaxonDescription td = this.getTaxonDescription(taxon, state
2643                                         .getConfig().getSourceReference(), false, true);
2644                         TextData citation = TextData.NewInstance(Feature.CITATION());
2645                         // TODO name used in source
2646                         citation.addSource(null, null, reference, microCitation);
2647                         td.addElement(citation);
2648                 } else if (TYPE.equalsIgnoreCase(classValue)) {
2649                         handleNotYetImplementedAttributeValue(parentEvent, CLASS,
2650                                         classValue);
2651                 } else {
2652                         // TODO Not yet implemented
2653                         handleNotYetImplementedAttributeValue(parentEvent, CLASS,
2654                                         classValue);
2655                 }
2656         }
2657
2658         private void handleFeature(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
2659                 String classValue = getClassOnlyAttribute(parentEvent);
2660                 Feature feature = makeFeature(classValue, state, parentEvent);
2661                 Taxon taxon = state.getCurrentTaxon();
2662                 TaxonDescription taxonDescription = getTaxonDescription(taxon, state.getConfig().getSourceReference(), NO_IMAGE_GALLERY, CREATE_NEW);
2663                 // TextData figureHolderTextData = null; //for use with one TextData for
2664                 // all figure only
2665
2666                 boolean isDescription = feature.equals(Feature.DESCRIPTION());
2667                 DescriptionElementBase lastDescriptionElement = null;
2668
2669                 while (reader.hasNext()) {
2670                         XMLEvent next = readNoWhitespace(reader);
2671                         if (isMyEndingElement(next, parentEvent)) {
2672                                 return;
2673                         } else if (isEndingElement(next, DISTRIBUTION_LIST) || isEndingElement(next, HABITAT_LIST)) {
2674                                 // only handle list elements
2675                         } else if (isStartingElement(next, HEADING)) {
2676                                 makeFeatureHeading(state, reader, classValue, feature, next);
2677                         } else if (isStartingElement(next, WRITER)) {
2678                                 makeFeatureWriter(state, reader, feature, taxon, next);
2679 //                      } else if (isStartingElement(next, DISTRIBUTION_LOCALITY)) {
2680 //                              if (!feature.equals(Feature.DISTRIBUTION())) {
2681 //                                      String message = "Distribution locality only allowed for feature of type 'distribution'";
2682 //                                      fireWarningEvent(message, next, 4);
2683 //                              }
2684 //                              handleDistributionLocality(state, reader, next);
2685                         } else if (isStartingElement(next, DISTRIBUTION_LIST) || isStartingElement(next, HABITAT_LIST)) {
2686                                 // only handle single list elements
2687                         } else if (isStartingElement(next, HABITAT)) {
2688                                 if (!(feature.equals(Feature.HABITAT())
2689                                                 || feature.equals(Feature.HABITAT_ECOLOGY())
2690                                                 || feature.equals(Feature.ECOLOGY()))) {
2691                                         String message = "Habitat only allowed for feature of type 'habitat','habitat ecology' or 'ecology'";
2692                                         fireWarningEvent(message, next, 4);
2693                                 }
2694                                 handleHabitat(state, reader, next);
2695                         } else if (isStartingElement(next, CHAR)) {
2696                                 TextData textData = handleChar(state, reader, next);
2697                                 taxonDescription.addElement(textData);
2698                         } else if (isStartingElement(next, STRING)) {
2699                                 lastDescriptionElement = makeFeatureString(state, reader,feature, taxonDescription, lastDescriptionElement,next);
2700                         } else if (isStartingElement(next, FIGURE_REF)) {
2701                                 lastDescriptionElement = makeFeatureFigureRef(state, reader, taxonDescription, isDescription, lastDescriptionElement, next);
2702                         } else if (isStartingElement(next, REFERENCES)) {
2703                                 // TODO details/microcitation ??
2704
2705                                 List<Reference<?>> refs = handleReferences(state, reader, next);
2706                                 if (!refs.isEmpty()) {
2707                                         // TODO
2708                                         Reference<?> descriptionRef = state.getConfig().getSourceReference();
2709                                         TaxonDescription description = getTaxonDescription(taxon, descriptionRef, false, true);
2710                                         TextData featurePlaceholder = getFeaturePlaceholder(state, description, feature, true);
2711                                         for (Reference<?> citation : refs) {
2712                                                 featurePlaceholder.addSource(null, null, citation, null);
2713                                         }
2714                                 } else {
2715                                         String message = "No reference found in references";
2716                                         fireWarningEvent(message, next, 6);
2717                                 }
2718                         } else if (isStartingElement(next, NUM)) {
2719                                 //TODO
2720                                 handleNotYetImplementedElement(next);
2721                         } else if (isEndingElement(next, NUM)) {
2722                                 //TODO
2723                                 popUnimplemented(next.asEndElement());
2724                         } else {
2725                                 handleUnexpectedElement(next);
2726                         }
2727                 }
2728                 throw new IllegalStateException("<Feature> has no closing tag");
2729         }
2730
2731         /**
2732          * @param state
2733          * @param reader
2734          * @param taxonDescription
2735          * @param isDescription
2736          * @param lastDescriptionElement
2737          * @param next
2738          * @return
2739          * @throws XMLStreamException
2740          */
2741         private DescriptionElementBase makeFeatureFigureRef(MarkupImportState state, XMLEventReader reader,TaxonDescription taxonDescription,
2742                                         boolean isDescription, DescriptionElementBase lastDescriptionElement, XMLEvent next)throws XMLStreamException {
2743                 FigureDataHolder figureHolder = handleFigureRef(state, reader, next);
2744                 Feature figureFeature = getFeature(state,MarkupTransformer.uuidFigures, "Figures", "Figures", "Fig.",null);
2745                 if (isDescription) {
2746                         TextData figureHolderTextData = null;
2747                         // if (figureHolderTextData == null){
2748                         figureHolderTextData = TextData.NewInstance(figureFeature);
2749                         if (StringUtils.isNotBlank(figureHolder.num)) {
2750                                 String annotationText = "<num>" + figureHolder.num.trim()
2751                                                 + "</num>";
2752                                 Annotation annotation = Annotation.NewInstance(annotationText,
2753                                                 AnnotationType.TECHNICAL(), Language.DEFAULT());
2754                                 figureHolderTextData.addAnnotation(annotation);
2755                         }
2756                         if (StringUtils.isNotBlank(figureHolder.figurePart)) {
2757                                 String annotationText = "<figurePart>"+ figureHolder.figurePart.trim() + "</figurePart>";
2758                                 Annotation annotation = Annotation.NewInstance(annotationText,AnnotationType.EDITORIAL(), Language.DEFAULT());
2759                                 figureHolderTextData.addAnnotation(annotation);
2760                         }
2761                         // if (StringUtils.isNotBlank(figureText)){
2762                         // figureHolderTextData.putText(Language.DEFAULT(), figureText);
2763                         // }
2764                         taxonDescription.addElement(figureHolderTextData);
2765                         // }
2766                         registerFigureDemand(state, figureHolderTextData, figureHolder.ref);
2767                 } else {
2768                         if (lastDescriptionElement == null) {
2769                                 String message = "No description element created yet that can be referred by figure. Create new TextData instead";
2770                                 fireWarningEvent(message, next, 4);
2771                                 lastDescriptionElement = TextData.NewInstance(figureFeature);
2772                                 taxonDescription.addElement(lastDescriptionElement);
2773                         }
2774                         registerFigureDemand(state, lastDescriptionElement,
2775                                         figureHolder.ref);
2776                 }
2777                 return lastDescriptionElement;
2778         }
2779
2780         /**
2781          * @param state
2782          * @param reader
2783          * @param feature
2784          * @param taxonDescription
2785          * @param lastDescriptionElement
2786          * @param distributionList
2787          * @param next
2788          * @return
2789          * @throws XMLStreamException
2790          */
2791         private DescriptionElementBase makeFeatureString(MarkupImportState state,XMLEventReader reader, Feature feature,
2792                                 TaxonDescription taxonDescription, DescriptionElementBase lastDescriptionElement, XMLEvent next) throws XMLStreamException {
2793                 Map<String, String> subheadingMap = handleString(state, reader, next, feature);
2794                 for (String subheading : subheadingMap.keySet()) {
2795                         Feature subheadingFeature = feature;
2796                         if (StringUtils.isNotBlank(subheading) && subheadingMap.size() > 1) {
2797                                 subheadingFeature = makeFeature(subheading, state, next);
2798                         }
2799                         TextData textData = TextData.NewInstance(subheadingFeature);
2800                         textData.putText(Language.DEFAULT(), subheadingMap.get(subheading));
2801                         taxonDescription.addElement(textData);
2802                         // TODO how to handle figures when these data are split in
2803                         // subheadings
2804                         lastDescriptionElement = textData;
2805                 }
2806                 return lastDescriptionElement;
2807         }
2808
2809         /**
2810          * @param state
2811          * @param reader
2812          * @param feature
2813          * @param taxon
2814          * @param next
2815          * @throws XMLStreamException
2816          */
2817         private void makeFeatureWriter(MarkupImportState state,XMLEventReader reader, Feature feature, Taxon taxon, XMLEvent next) throws XMLStreamException {
2818                 WriterDataHolder writer = handleWriter(state, reader, next);
2819                 if (isNotBlank(writer.writer)) {
2820                         // TODO
2821                         Reference<?> ref = state.getConfig().getSourceReference();
2822                         TaxonDescription description = getTaxonDescription(taxon, ref,
2823                                         false, true);
2824                         TextData featurePlaceholder = getFeaturePlaceholder(state,
2825                                         description, feature, true);
2826                         featurePlaceholder.addAnnotation(writer.annotation);
2827                         registerFootnotes(state, featurePlaceholder, writer.footnotes);
2828                 } else {
2829                         String message = "Writer element is empty";
2830                         fireWarningEvent(message, next, 4);
2831                 }
2832         }
2833
2834         /**
2835          * @param state
2836          * @param reader
2837          * @param classValue
2838          * @param feature
2839          * @param next
2840          * @throws XMLStreamException
2841          */
2842         private void makeFeatureHeading(MarkupImportState state, XMLEventReader reader, String classValue, Feature feature, XMLEvent next) throws XMLStreamException {
2843                 String heading = handleHeading(state, reader, next);
2844                 if (StringUtils.isNotBlank(heading)) {
2845                         if (!heading.equalsIgnoreCase(classValue)) {
2846                                 try {
2847                                         if (!feature.equals(state.getTransformer().getFeatureByKey(
2848                                                         heading))) {
2849                                                 UUID headerFeatureUuid = state.getTransformer()
2850                                                                 .getFeatureUuid(heading);
2851                                                 if (!feature.getUuid().equals(headerFeatureUuid)) {
2852                                                         String message = "Feature heading '%s' differs from feature class '%s' and can not be transformed to feature";
2853                                                         message = String.format(message, heading,
2854                                                                         classValue);
2855                                                         fireWarningEvent(message, next, 1);
2856                                                 }
2857                                         }
2858                                 } catch (UndefinedTransformerMethodException e) {
2859                                         throw new RuntimeException(e);
2860                                 }
2861                         } else {
2862                                 // do nothing
2863                         }
2864                 }
2865         }
2866
2867         private List<Reference<?>> handleReferences(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
2868                 // attributes
2869                 Map<String, Attribute> attributes = getAttributes(parentEvent);
2870                 String bibliography = getAndRemoveAttributeValue(attributes,
2871                                 BIBLIOGRAPHY);
2872                 String serialsAbbreviations = getAndRemoveAttributeValue(attributes,
2873                                 SERIALS_ABBREVIATIONS);
2874                 if (isNotBlank(bibliography) || isNotBlank(serialsAbbreviations)) {
2875                         String message = "Attributes not yet implemented for <references>";
2876                         fireWarningEvent(message, parentEvent, 4);
2877                 }
2878
2879                 List<Reference<?>> result = new ArrayList<Reference<?>>();
2880
2881                 // elements
2882                 while (reader.hasNext()) {
2883                         XMLEvent next = readNoWhitespace(reader);
2884                         if (next.isEndElement()) {
2885                                 if (isMyEndingElement(next, parentEvent)) {
2886                                         return result;
2887                                 } else {
2888                                         if (isEndingElement(next, HEADING)) {
2889                                                 // NOT YET IMPLEMENTED
2890                                                 popUnimplemented(next.asEndElement());
2891                                         } else if (isEndingElement(next, WRITER)) {
2892                                                 // NOT YET IMPLEMENTED
2893                                                 popUnimplemented(next.asEndElement());
2894                                         } else if (isEndingElement(next, FOOTNOTE)) {
2895                                                 // NOT YET IMPLEMENTED
2896                                                 popUnimplemented(next.asEndElement());
2897                                         } else if (isEndingElement(next, STRING)) {
2898                                                 // NOT YET IMPLEMENTED
2899                                                 popUnimplemented(next.asEndElement());
2900                                         } else if (isEndingElement(next, REF_NUM)) {
2901                                                 // NOT YET IMPLEMENTED
2902                                                 popUnimplemented(next.asEndElement());
2903                                         } else {
2904                                                 handleUnexpectedEndElement(next.asEndElement());
2905                                         }
2906                                 }
2907                         } else if (next.isStartElement()) {
2908                                 if (isStartingElement(next, HEADING)) {
2909                                         handleNotYetImplementedElement(next);
2910                                 } else if (isStartingElement(next, SUB_HEADING)) {
2911                                         String subheading = getCData(state, reader, next).trim();
2912                                         String excludePattern = "(i?)(References?|Literature):?";
2913                                         if (!subheading.matches(excludePattern)) {
2914                                                 fireNotYetImplementedElement(next.getLocation(), next.asStartElement().getName(), 0);
2915                                         }
2916                                 } else if (isStartingElement(next, WRITER)) {
2917                                         handleNotYetImplementedElement(next);
2918                                 } else if (isStartingElement(next, FOOTNOTE)) {
2919                                         handleNotYetImplementedElement(next);
2920                                 } else if (isStartingElement(next, STRING)) {
2921                                         handleNotYetImplementedElement(next);
2922                                 } else if (isStartingElement(next, REF_NUM)) {
2923                                         handleNotYetImplementedElement(next);
2924                                 } else if (isStartingElement(next, REFERENCE)) {
2925                                         Reference<?> ref = handleReference(state, reader, next);
2926                                         result.add(ref);
2927                                 } else {
2928                                         handleUnexpectedStartElement(next);
2929                                 }
2930                         } else {
2931                                 handleUnexpectedElement(next);
2932                         }
2933                 }
2934                 throw new IllegalStateException("<References> has no closing tag");
2935         }
2936
2937         private void handleHabitat(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
2938                 checkNoAttributes(parentEvent);
2939                 Taxon taxon = state.getCurrentTaxon();
2940                 // TODO which ref to take?
2941                 Reference<?> ref = state.getConfig().getSourceReference();
2942
2943                 String text = "";
2944                 while (reader.hasNext()) {
2945                         XMLEvent next = readNoWhitespace(reader);
2946                         if (isMyEndingElement(next, parentEvent)) {
2947                                 TaxonDescription description = getTaxonDescription(taxon, ref,
2948                                                 false, true);
2949                                 UUID uuidExtractedHabitat = MarkupTransformer.uuidExtractedHabitat;
2950                                 Feature feature = getFeature(
2951                                                 state,
2952                                                 uuidExtractedHabitat,
2953                                                 "Extracted Habitat",
2954                                                 "An structured habitat that was extracted from a habitat text",
2955                                                 "extr. habit.", null);
2956                                 TextData habitat = TextData.NewInstance(feature);
2957                                 habitat.putText(Language.DEFAULT(), text);
2958                                 description.addElement(habitat);
2959
2960                                 return;
2961                         } else if (next.isStartElement()) {
2962                                 if (isStartingElement(next, ALTITUDE)) {
2963                                         text = text.trim() + getTaggedCData(state, reader, next);
2964                                 } else if (isStartingElement(next, LIFE_CYCLE_PERIODS)) {
2965                                         handleNotYetImplementedElement(next);
2966                                 } else {
2967                                         handleUnexpectedStartElement(next.asStartElement());
2968                                 }
2969                         } else if (next.isCharacters()) {
2970                                 text += next.asCharacters().getData();
2971                         } else {
2972                                 handleUnexpectedElement(next);
2973                         }
2974                 }
2975                 throw new IllegalStateException("<Habitat> has no closing tag");
2976         }
2977
2978         private String getTaggedCData(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
2979                 checkNoAttributes(parentEvent);
2980
2981                 String text = getXmlTag(parentEvent);
2982                 while (reader.hasNext()) {
2983                         XMLEvent next = readNoWhitespace(reader);
2984                         if (isMyEndingElement(next, parentEvent)) {
2985                                 text += getXmlTag(next);
2986                                 return text;
2987                         } else if (next.isStartElement()) {
2988                                 text += getTaggedCData(state, reader, next);
2989                         } else if (next.isEndElement()) {
2990                                 text += getTaggedCData(state, reader, next);
2991                         } else if (next.isCharacters()) {
2992                                 text += next.asCharacters().getData();
2993                         } else {
2994                                 handleUnexpectedEndElement(next.asEndElement());
2995                         }
2996                 }
2997                 throw new IllegalStateException("Some tag has no closing tag");
2998         }
2999
3000         private String handleDistributionLocality(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent)throws XMLStreamException {
3001                 Map<String, Attribute> attributes = getAttributes(parentEvent);
3002                 String classValue = getAndRemoveRequiredAttributeValue(parentEvent, attributes, CLASS);
3003                 String statusValue =getAndRemoveAttributeValue(attributes, STATUS);
3004                 String frequencyValue =getAndRemoveAttributeValue(attributes, FREQUENCY);
3005
3006
3007                 Taxon taxon = state.getCurrentTaxon();
3008                 // TODO which ref to take?
3009                 Reference<?> ref = state.getConfig().getSourceReference();
3010
3011                 String text = "";
3012                 while (reader.hasNext()) {
3013                         XMLEvent next = readNoWhitespace(reader);
3014                         if (isMyEndingElement(next, parentEvent)) {
3015                                 if (StringUtils.isNotBlank(text)) {
3016                                         String label = CdmUtils.removeTrailingDot(normalize(text));
3017                                         TaxonDescription description = getTaxonDescription(taxon, ref, false, true);
3018                                         NamedAreaLevel level = makeNamedAreaLevel(state,classValue, next);
3019
3020                                         //status
3021                                         PresenceAbsenceTermBase<?> status = null;
3022                                         if (isNotBlank(statusValue)){
3023                                                 try {
3024                                                         status = state.getTransformer().getPresenceTermByKey(statusValue);
3025                                                         if (status == null){
3026                                                                 //TODO
3027                                                                 String message = "The status '%s' could not be transformed to an CDM status";
3028                                                                 fireWarningEvent(message, next, 4);
3029                                                         }
3030                                                 } catch (UndefinedTransformerMethodException e) {
3031                                                         throw new RuntimeException(e);
3032                                                 }
3033                                         }else{
3034                                                 status = PresenceTerm.PRESENT();
3035                                         }
3036                                         //frequency
3037                                         if (isNotBlank(frequencyValue)){
3038                                                 String message = "The frequency attribute is currently not yet available in CDM";
3039                                                 fireWarningEvent(message, parentEvent, 6);
3040                                         }
3041
3042                                         NamedArea higherArea = null;
3043                                         List<NamedArea> areas = new ArrayList<NamedArea>();
3044
3045                                         String patSingleArea = "([^,\\(]{3,})";
3046                                         String patSeparator = "(,|\\sand\\s)";
3047                                         String hierarchiePattern = String.format("%s\\((%s(%s%s)*)\\)",patSingleArea, patSingleArea, patSeparator, patSingleArea);
3048                                         Pattern patHierarchie = Pattern.compile(hierarchiePattern, Pattern.CASE_INSENSITIVE);
3049                                         Matcher matcher = patHierarchie.matcher(label);
3050                                         if (matcher.matches()){
3051                                                 String higherAreaStr = matcher.group(1).trim();
3052                                                 higherArea =  makeArea(state, higherAreaStr, level);
3053                                                 String[] innerAreas = matcher.group(2).split(patSeparator);
3054                                                 for (String innerArea : innerAreas){
3055                                                         if (isNotBlank(innerArea)){
3056                                                                 NamedArea singleArea = makeArea(state, innerArea.trim(), level);
3057                                                                 areas.add(singleArea);
3058                                                                 NamedArea partOf = singleArea.getPartOf();
3059 //                                                              if (partOf == null){
3060 //                                                                      singleArea.setPartOf(higherArea);
3061 //                                                              }
3062                                                         }
3063                                                 }
3064                                         }else{
3065                                                 NamedArea singleArea = makeArea(state, label, level);
3066                                                 areas.add(singleArea);
3067                                         }
3068
3069                                         for (NamedArea area : areas){
3070                                                 //create distribution
3071                                                 Distribution distribution = Distribution.NewInstance(area,status);
3072                                                 description.addElement(distribution);
3073                                         }
3074                                 } else {
3075                                         String message = "Empty distribution locality";
3076                                         fireWarningEvent(message, next, 4);
3077                                 }
3078                                 return text;
3079                         } else if (isStartingElement(next, COORDINATES)) {
3080                                 //TODO
3081                                 handleNotYetImplementedElement(next);
3082                         } else if (isEndingElement(next, COORDINATES)) {
3083                                 //TODO
3084                                 popUnimplemented(next.asEndElement());
3085                         } else if (next.isCharacters()) {
3086                                 text += next.asCharacters().getData();
3087                         } else {
3088                                 handleUnexpectedEndElement(next.asEndElement());
3089                         }
3090                 }
3091                 throw new IllegalStateException("<DistributionLocality> has no closing tag");
3092         }
3093
3094         /**
3095          * @param state
3096          * @param areaName
3097          * @param level
3098          * @return
3099          */
3100         private NamedArea makeArea(MarkupImportState state, String areaName, NamedAreaLevel level) {
3101
3102
3103                 //TODO FM vocabulary
3104                 TermVocabulary<NamedArea> voc = null;
3105                 NamedAreaType areaType = null;
3106
3107                 NamedArea area = null;
3108                 try {
3109                         area = state.getTransformer().getNamedAreaByKey(areaName);
3110                 } catch (UndefinedTransformerMethodException e) {
3111                         throw new RuntimeException(e);
3112                 }
3113                 if (area == null){
3114                         boolean isNewInState = false;
3115                         UUID uuid = state.getAreaUuid(areaName);
3116                         if (uuid == null){
3117                                 isNewInState = true;
3118
3119
3120                                 try {
3121                                         uuid = state.getTransformer().getNamedAreaUuid(areaName);
3122                                 } catch (UndefinedTransformerMethodException e) {
3123                                         throw new RuntimeException(e);
3124                                 }
3125                         }
3126                         TermMatchMode matchMode = TermMatchMode.UUID_LABEL;
3127                         area = getNamedArea(state, uuid, areaName, areaName, areaName, areaType, level, voc, matchMode);
3128                         if (isNewInState){
3129                                 state.putAreaUuid(areaName, area.getUuid());
3130
3131                                 //TODO just for testing -> make generic and move to better place
3132                                 String geoServiceLayer="vmap0_as_bnd_political_boundary_a";
3133                                 String layerFieldName ="nam";
3134
3135                                 if ("Bangka".equals(areaName)){
3136                                         String areaValue = "PULAU BANGKA#SUMATERA SELATAN";
3137                                         GeoServiceArea geoServiceArea = new GeoServiceArea();
3138                                         geoServiceArea.add(geoServiceLayer, layerFieldName, areaValue);
3139                                         this.editGeoService.setMapping(area, geoServiceArea);
3140 //                                      save(area, state);
3141                                 }
3142                                 if ("Luzon".equals(areaName)){
3143                                         GeoServiceArea geoServiceArea = new GeoServiceArea();
3144
3145                                         List<String> list = Arrays.asList("HERMANA MAYOR ISLAND#CENTRAL LUZON",
3146                                                         "HERMANA MENOR ISLAND#CENTRAL LUZON",
3147                                                         "CENTRAL LUZON");
3148                                         for (String areaValue : list){
3149                                                 geoServiceArea.add(geoServiceLayer, layerFieldName, areaValue);
3150                                         }
3151
3152                                         this.editGeoService.setMapping(area, geoServiceArea);
3153 //                                      save(area, state);
3154                                 }
3155                                 if ("Mindanao".equals(areaName)){
3156                                         GeoServiceArea geoServiceArea = new GeoServiceArea();
3157
3158                                         List<String> list = Arrays.asList("NORTHERN MINDANAO",
3159                                                         "SOUTHERN MINDANAO",
3160                                                         "WESTERN MINDANAO");
3161                                         //TODO to be continued
3162                                         for (String areaValue : list){
3163                                                 geoServiceArea.add(geoServiceLayer, layerFieldName, areaValue);
3164                                         }
3165
3166                                         this.editGeoService.setMapping(area, geoServiceArea);
3167 //                                      save(area, state);
3168                                 }
3169                                 if ("Palawan".equals(areaName)){
3170                                         GeoServiceArea geoServiceArea = new GeoServiceArea();
3171
3172                                         List<String> list = Arrays.asList("PALAWAN#SOUTHERN TAGALOG");
3173                                         for (String areaValue : list){
3174                                                 geoServiceArea.add(geoServiceLayer, layerFieldName, areaValue);
3175                                         }
3176
3177                                         this.editGeoService.setMapping(area, geoServiceArea);
3178 //                                      save(area, state);
3179                                 }
3180
3181
3182                         }
3183                 }
3184                 return area;
3185         }
3186
3187
3188         /**
3189          * @param state
3190          * @param levelString
3191          * @param next
3192          * @return
3193          */
3194         private NamedAreaLevel makeNamedAreaLevel(MarkupImportState state,
3195                         String levelString, XMLEvent next) {
3196                 NamedAreaLevel level;
3197                 try {
3198                         level = state.getTransformer().getNamedAreaLevelByKey(levelString);
3199                         if (level == null) {
3200                                 UUID levelUuid = state.getTransformer().getNamedAreaLevelUuid(levelString);
3201                                 if (levelUuid == null) {
3202                                         String message = "Unknown distribution locality class (named area level): %s. Create new level instead.";
3203                                         message = String.format(message, levelString);
3204                                         fireWarningEvent(message, next, 6);
3205                                 }
3206                                 level = getNamedAreaLevel(state, levelUuid, levelString,
3207                                                 levelString, levelString, null);
3208                         }
3209                 } catch (UndefinedTransformerMethodException e) {
3210                         throw new RuntimeException(e);
3211                 }
3212                 return level;
3213         }
3214
3215         private String handleHeading(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent)throws XMLStreamException {
3216                 checkNoAttributes(parentEvent);
3217
3218                 String text = "";
3219                 while (reader.hasNext()) {
3220                         XMLEvent next = readNoWhitespace(reader);
3221                         if (isMyEndingElement(next, parentEvent)) {
3222                                 return text;
3223                         } else if (next.isStartElement()) {
3224                                 if (isStartingElement(next, FOOTNOTE)) {
3225                                         handleNotYetImplementedElement(next);
3226                                 } else {
3227                                         handleUnexpectedStartElement(next.asStartElement());
3228                                 }
3229                         } else if (next.isCharacters()) {
3230                                 text += next.asCharacters().getData();
3231                         } else {
3232                                 handleUnexpectedEndElement(next.asEndElement());
3233                         }
3234                 }
3235                 throw new IllegalStateException("<String> has no closing tag");
3236
3237         }
3238
3239         /**
3240          * Handle string
3241          * @param state
3242          * @param reader
3243          * @param parentEvent
3244          * @param feature only needed for distributionLocalities
3245          * @return
3246          * @throws XMLStreamException
3247          */
3248         private Map<String, String> handleString(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent, Feature feature)throws XMLStreamException {
3249                 // attributes
3250                 String classValue = getClassOnlyAttribute(parentEvent, false);
3251                 if (StringUtils.isNotBlank(classValue)) {
3252                         String message = "class attribute for <string> not yet implemented";
3253                         fireWarningEvent(message, parentEvent, 2);
3254                 }
3255
3256                 // subheadings
3257                 Map<String, String> subHeadingMap = new HashMap<String, String>();
3258                 String currentSubheading = null;
3259
3260                 boolean isTextMode = true;
3261                 String text = "";
3262                 while (reader.hasNext()) {
3263                         XMLEvent next = readNoWhitespace(reader);
3264                         if (isMyEndingElement(next, parentEvent)) {
3265                                 putCurrentSubheading(subHeadingMap, currentSubheading, text);
3266                                 return subHeadingMap;
3267                         } else if (isStartingElement(next, BR)) {
3268                                 text += "<br/>";
3269                                 isTextMode = false;
3270                         } else if (isEndingElement(next, BR)) {
3271                                 isTextMode = true;
3272                         } else if (isHtml(next)) {
3273                                 text += getXmlTag(next);
3274                         } else if (isStartingElement(next, SUB_HEADING)) {
3275                                 text = putCurrentSubheading(subHeadingMap,currentSubheading, text);
3276                                 // TODO footnotes
3277                                 currentSubheading = getCData(state, reader, next).trim();
3278                         } else if (isStartingElement(next, DISTRIBUTION_LOCALITY)) {
3279                                 if (feature != null && !feature.equals(Feature.DISTRIBUTION())) {
3280                                         String message = "Distribution locality only allowed for feature of type 'distribution'";
3281                                         fireWarningEvent(message, next, 4);
3282                                 }
3283                                 text += handleDistributionLocality(state, reader, next);
3284                         } else if (next.isCharacters()) {
3285                                 if (!isTextMode) {
3286                                         String message = "String is not in text mode";
3287                                         fireWarningEvent(message, next, 6);
3288                                 } else {
3289                                         text += next.asCharacters().getData();
3290                                 }
3291                         } else if (isStartingElement(next, HEADING)) {
3292                                 //TODO
3293                                 handleNotYetImplementedElement(next);
3294                         } else if (isEndingElement(next, HEADING)) {
3295                                 //TODO
3296                                 popUnimplemented(next.asEndElement());
3297                         } else if (isStartingElement(next, QUOTE)) {
3298                                 //TODO
3299                                 handleNotYetImplementedElement(next);
3300                         } else if (isEndingElement(next, QUOTE)) {
3301                                 //TODO
3302                                 popUnimplemented(next.asEndElement());
3303                         } else if (isStartingElement(next, DEDICATION)) {
3304                                 //TODO
3305                                 handleNotYetImplementedElement(next);
3306                         } else if (isEndingElement(next, DEDICATION)) {
3307                                 //TODO
3308                                 popUnimplemented(next.asEndElement());
3309                         } else if (isStartingElement(next, TAXONTYPE)) {
3310                                 //TODO
3311                                 handleNotYetImplementedElement(next);
3312                         } else if (isEndingElement(next, TAXONTYPE)) {
3313                                 //TODO
3314                                 popUnimplemented(next.asEndElement());
3315                         } else if (isStartingElement(next, FULL_NAME)) {
3316                                 //TODO
3317                                 handleNotYetImplementedElement(next);
3318                         } else if (isEndingElement(next, FULL_NAME)) {
3319                                 //TODO
3320                                 popUnimplemented(next.asEndElement());
3321                         }else if (isStartingElement(next, REFERENCES)) {
3322                                 //TODO
3323                                 handleNotYetImplementedElement(next);
3324                         } else if (isEndingElement(next, REFERENCES)) {
3325                                 //TODO
3326                                 popUnimplemented(next.asEndElement());
3327                         } else if (isStartingElement(next, GATHERING)) {
3328                                 //TODO
3329                                 handleNotYetImplementedElement(next);
3330                         } else if (isEndingElement(next, GATHERING)) {
3331                                 //TODO
3332                                 popUnimplemented(next.asEndElement());
3333                         } else if (isStartingElement(next, ANNOTATION)) {
3334                                 //TODO
3335                                 handleNotYetImplementedElement(next);
3336                         } else if (isEndingElement(next, ANNOTATION)) {
3337                                 //TODO
3338                                 popUnimplemented(next.asEndElement());
3339                         } else if (isStartingElement(next, HABITAT)) {
3340                                 //TODO
3341                                 handleNotYetImplementedElement(next);
3342                         } else if (isEndingElement(next, HABITAT)) {
3343                                 //TODO
3344                                 popUnimplemented(next.asEndElement());
3345                         } else if (isStartingElement(next, FIGURE_REF)) {
3346                                 //TODO
3347                                 handleNotYetImplementedElement(next);
3348                         } else if (isEndingElement(next, FIGURE_REF)) {
3349                                 //TODO
3350                                 popUnimplemented(next.asEndElement());
3351                         } else if (isStartingElement(next, FIGURE)) {
3352                                 //TODO
3353                                 handleNotYetImplementedElement(next);
3354                         } else if (isEndingElement(next, FIGURE)) {
3355                                 //TODO
3356                                 popUnimplemented(next.asEndElement());
3357                         } else if (isStartingElement(next, FOOTNOTE_REF)) {
3358                                 //TODO
3359                                 handleNotYetImplementedElement(next);
3360                         } else if (isEndingElement(next, FOOTNOTE_REF)) {
3361                                 //TODO
3362                                 popUnimplemented(next.asEndElement());
3363                         } else if (isStartingElement(next, FOOTNOTE)) {
3364                                 //TODO
3365                                 handleNotYetImplementedElement(next);
3366                         } else if (isEndingElement(next, FOOTNOTE)) {
3367                                 //TODO
3368                                 popUnimplemented(next.asEndElement());
3369                         } else if (isStartingElement(next, WRITER)) {
3370                                 //TODO
3371                                 handleNotYetImplementedElement(next);
3372                         } else if (isEndingElement(next, WRITER)) {
3373                                 //TODO
3374                                 popUnimplemented(next.asEndElement());
3375                         } else if (isStartingElement(next, DATES)) {
3376                                 //TODO
3377                                 handleNotYetImplementedElement(next);
3378                         } else if (isEndingElement(next, DATES)) {
3379                                 //TODO
3380                                 popUnimplemented(next.asEndElement());
3381                         } else {
3382                                 handleUnexpectedElement(next);
3383                         }
3384                 }
3385                 throw new IllegalStateException("<String> has no closing tag");
3386         }
3387
3388         /**
3389          * @param subHeadingMap
3390          * @param currentSubheading
3391          * @param text
3392          * @return
3393          */
3394         private String putCurrentSubheading(Map<String, String> subHeadingMap, String currentSubheading, String text) {
3395                 if (StringUtils.isNotBlank(text)) {
3396                         text = removeStartingMinus(text);
3397                         subHeadingMap.put(currentSubheading, text.trim());
3398                 }
3399                 return "";
3400         }
3401
3402         private String removeStartingMinus(String string) {
3403                 string = replaceStart(string, "-");
3404                 string = replaceStart(string, "\u002d");
3405                 string = replaceStart(string, "\u2013");
3406                 string = replaceStart(string, "\u2014");
3407                 string = replaceStart(string, "--");
3408                 return string;
3409         }
3410
3411         /**
3412          * @param value
3413          * @param replacementString
3414          */
3415         private String replaceStart(String value, String replacementString) {
3416                 if (value.startsWith(replacementString) ){
3417                         value = value.substring(replacementString.length()).trim();
3418                 }
3419                 while (value.startsWith("-") || value.startsWith("\u2014") ){
3420                         value = value.substring("-".length()).trim();
3421                 }
3422                 return value;
3423         }
3424
3425         private String getXmlTag(XMLEvent event) {
3426                 String result;
3427                 if (event.isStartElement()) {
3428                         result = "<" + event.asStartElement().getName().getLocalPart()
3429                                         + ">";
3430                 } else if (event.isEndElement()) {
3431                         result = "</" + event.asEndElement().getName().getLocalPart() + ">";
3432                 } else {
3433                         String message = "Only start or end elements are allowed as Html tags";
3434                         throw new IllegalStateException(message);
3435                 }
3436                 return result;
3437         }
3438
3439         protected static final List<String> htmlList = Arrays.asList("sub", "sup",
3440                         "ol", "ul", "li", "i", "b", "table", "br");
3441
3442         private boolean isHtml(XMLEvent event) {
3443                 if (event.isStartElement()) {
3444                         String tag = event.asStartElement().getName().getLocalPart();
3445                         return htmlList.contains(tag);
3446                 } else if (event.isEndElement()) {
3447                         String tag = event.asEndElement().getName().getLocalPart();
3448                         return htmlList.contains(tag);
3449                 } else {
3450                         return false;
3451                 }
3452
3453         }
3454
3455         private TextData handleChar(MarkupImportState state, XMLEventReader reader,
3456                         XMLEvent parentEvent) throws XMLStreamException {
3457                 String classValue = getClassOnlyAttribute(parentEvent);
3458                 Feature feature = makeFeature(classValue, state, parentEvent);
3459
3460                 String text = "";
3461                 while (reader.hasNext()) {
3462                         XMLEvent next = readNoWhitespace(reader);
3463                         if (isMyEndingElement(next, parentEvent)) {
3464                                 TextData textData = TextData.NewInstance(feature);
3465                                 textData.putText(Language.DEFAULT(), text);
3466                                 return textData;
3467                         } else if (isStartingElement(next, FIGURE_REF)) {
3468                                 //TODO
3469                                 handleNotYetImplementedElement(next);
3470                         } else if (isEndingElement(next, FIGURE_REF)) {
3471                                 //TODO
3472                                 popUnimplemented(next.asEndElement());
3473                         } else if (next.isStartElement()) {
3474                                 if (isStartingElement(next, ANNOTATION)) {
3475                                         handleNotYetImplementedElement(next);
3476                                 } else if (isStartingElement(next, ITALICS)) {
3477                                         handleNotYetImplementedElement(next);
3478                                 } else if (isStartingElement(next, BOLD)) {
3479                                         handleNotYetImplementedElement(next);
3480                                 } else {
3481                                         handleUnexpectedStartElement(next.asStartElement());
3482                                 }
3483                         } else if (next.isCharacters()) {
3484                                 text += next.asCharacters().getData();
3485                         } else {
3486                                 handleUnexpectedEndElement(next.asEndElement());
3487                         }
3488                 }
3489                 throw new IllegalStateException("RefPart has no closing tag");
3490         }
3491
3492         /**
3493          * @param classValue
3494          * @param state
3495          * @param parentEvent
3496          * @return
3497          * @throws UndefinedTransformerMethodException
3498          */
3499         private Feature makeFeature(String classValue, MarkupImportState state, XMLEvent parentEvent) {
3500                 UUID uuid;
3501                 try {
3502                         Feature feature = state.getTransformer().getFeatureByKey(classValue);
3503                         if (feature != null) {
3504                                 return feature;
3505                         }
3506                         uuid = state.getTransformer().getFeatureUuid(classValue);
3507                         if (uuid == null) {
3508                                 // TODO
3509                                 String message = "Uuid is not defined for '%s'";
3510                                 message = String.format(message, classValue);
3511                                 fireWarningEvent(message, parentEvent, 8);
3512                         }
3513                         String featureText = StringUtils.capitalize(classValue);
3514
3515                         // TODO eFlora vocabulary
3516                         TermVocabulary<Feature> voc = null;
3517                         feature = getFeature(state, uuid, featureText, featureText, classValue, voc);
3518                         if (feature == null) {
3519                                 throw new NullPointerException(classValue + " not recognized as a feature");
3520                         }
3521                         return feature;
3522                 } catch (Exception e) {
3523                         String message = "Could not create feature for %s: %s";
3524                         message = String.format(message, classValue, e.getMessage());
3525                         fireWarningEvent(message, parentEvent, 4);
3526                         return Feature.UNKNOWN();
3527                 }
3528         }
3529
3530         /**
3531          * This comes from the old version, needs to be checked on need
3532          *
3533          * @param state
3534          */
3535         private void doAllTheOldOtherStuff(MarkupImportState state) {
3536                 state.putTree(null, null);
3537                 if (unmatchedLeads == null) {
3538                         unmatchedLeads = UnmatchedLeads.NewInstance();
3539                 }
3540                 state.setUnmatchedLeads(unmatchedLeads);
3541
3542                 // TransactionStatus tx = startTransaction();
3543                 unmatchedLeads.saveToSession(getPolytomousKeyNodeService());
3544
3545                 // TODO generally do not store the reference object in the config
3546                 Reference sourceReference = state.getConfig().getSourceReference();
3547                 getReferenceService().saveOrUpdate(sourceReference);
3548         }
3549
3550         /*
3551          * (non-Javadoc)
3552          *
3553          * @see
3554          * eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common
3555          * .IImportConfigurator)
3556          */
3557         protected boolean isIgnore(MarkupImportState state) {
3558                 return !state.getConfig().isDoTaxa();
3559         }
3560
3561 }