cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.markup;
  11
  12 import java.net.MalformedURLException;
  13 import java.util.ArrayList;
  14 import java.util.Arrays;
  15 import java.util.Collection;
  16 import java.util.HashMap;
  17 import java.util.HashSet;
  18 import java.util.Iterator;
  19 import java.util.List;
  20 import java.util.Map;
  21 import java.util.Set;
  22 import java.util.Stack;
  23 import java.util.UUID;
  24 import java.util.regex.Matcher;
  25 import java.util.regex.Pattern;
  26
  27 import javax.xml.namespace.QName;
  28 import javax.xml.stream.Location;
  29 import javax.xml.stream.XMLEventReader;
  30 import javax.xml.stream.XMLStreamConstants;
  31 import javax.xml.stream.XMLStreamException;
  32 import javax.xml.stream.events.Attribute;
  33 import javax.xml.stream.events.Characters;
  34 import javax.xml.stream.events.EndElement;
  35 import javax.xml.stream.events.StartElement;
  36 import javax.xml.stream.events.XMLEvent;
  37
  38 import org.apache.commons.lang.StringUtils;
  39 import org.apache.commons.lang.WordUtils;
  40 import org.apache.log4j.Logger;
  41
  42 import eu.etaxonomy.cdm.api.service.IClassificationService;
  43 import eu.etaxonomy.cdm.api.service.ITermService;
  44 import eu.etaxonomy.cdm.common.CdmUtils;
  45 import eu.etaxonomy.cdm.ext.geo.GeoServiceArea;
  46 import eu.etaxonomy.cdm.ext.geo.IEditGeoService;
  47 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  48 import eu.etaxonomy.cdm.io.common.CdmImportBase.TermMatchMode;
  49 import eu.etaxonomy.cdm.io.common.events.IIoEvent;
  50 import eu.etaxonomy.cdm.io.common.events.IoProblemEvent;
  51 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  52 import eu.etaxonomy.cdm.model.agent.Team;
  53 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  54 import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
  55 import eu.etaxonomy.cdm.model.common.Annotation;
  56 import eu.etaxonomy.cdm.model.common.AnnotationType;
  57 import eu.etaxonomy.cdm.model.common.CdmBase;
  58 import eu.etaxonomy.cdm.model.common.DefinedTerm;
  59 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  60 import eu.etaxonomy.cdm.model.common.Extension;
  61 import eu.etaxonomy.cdm.model.common.ExtensionType;
  62 import eu.etaxonomy.cdm.model.common.Language;
  63 import eu.etaxonomy.cdm.model.common.MarkerType;
  64 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  65 import eu.etaxonomy.cdm.model.common.TermVocabulary;
  66 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  67 import eu.etaxonomy.cdm.model.description.Distribution;
  68 import eu.etaxonomy.cdm.model.description.Feature;
  69 import eu.etaxonomy.cdm.model.description.PolytomousKey;
  70 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
  71 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  72 import eu.etaxonomy.cdm.model.description.TextData;
  73 import eu.etaxonomy.cdm.model.location.NamedArea;
  74 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  75 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  76 import eu.etaxonomy.cdm.model.media.IdentifiableMediaEntity;
  77 import eu.etaxonomy.cdm.model.media.Media;
  78 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  79 import eu.etaxonomy.cdm.model.name.NonViralName;
  80 import eu.etaxonomy.cdm.model.name.Rank;
  81 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  82 import eu.etaxonomy.cdm.model.reference.Reference;
  83 import eu.etaxonomy.cdm.model.taxon.Classification;
  84 import eu.etaxonomy.cdm.model.taxon.Taxon;
  85 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  86 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
  87 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  88
  89 /**
  90  * @author a.mueller
  91  * @created 04.08.2008
  92  */
  93 public abstract class MarkupImportBase  {
  94         private static final Logger logger = Logger.getLogger(MarkupImportBase.class);
  95
  96         //Base
  97         protected static final String ALTITUDE = "altitude";
  98         protected static final String ANNOTATION = "annotation";
  99         protected static final String BOLD = "bold";
 100         protected static final String BR = "br";
 101         protected static final String DOUBTFUL = "doubtful";
 102         protected static final String CITATION = "citation";
 103         protected static final String CLASS = "class";
 104         protected static final String COORDINATES = "coordinates";
 105         protected static final String DATES = "dates";
 106         protected static final String GATHERING = "gathering";
 107         protected static final String GATHERING_GROUP = "gatheringGroup";
 108         protected static final String GENUS_ABBREVIATION = "genus abbreviation";
 109         protected static final String FOOTNOTE = "footnote";
 110         protected static final String FOOTNOTE_REF = "footnoteRef";
 111         protected static final String FULL_NAME = "fullName";
 112         protected static final String ITALICS = "italics";
 113         protected static final String NUM = "num";
 114         protected static final String NOTES = "notes";
 115         protected static final String PUBLICATION = "publication";
 116         protected static final String SPECIMEN_TYPE = "specimenType";
 117         protected static final String STATUS = "status";
 118         protected static final String SUB_HEADING = "subHeading";
 119         protected static final String TYPE = "type";
 120         protected static final String TYPE_STATUS = "typeStatus";
 121         protected static final String UNKNOWN = "unknown";
 122
 123
 124         protected static final boolean CREATE_NEW = true;
 125         protected static final boolean NO_IMAGE_GALLERY = false;
 126         protected static final boolean IMAGE_GALLERY = true;
 127
 128         protected static final String ADDENDA = "addenda";
 129         protected static final String BIBLIOGRAPHY = "bibliography";
 130         protected static final String BIOGRAPHIES = "biographies";
 131         protected static final String CHAR = "char";
 132         protected static final String DEDICATION = "dedication";
 133         protected static final String DEFAULT_MEDIA_URL = "defaultMediaUrl";
 134         protected static final String DISTRIBUTION_LIST = "distributionList";
 135         protected static final String DISTRIBUTION_LOCALITY = "distributionLocality";
 136         protected static final String FEATURE = "feature";
 137         protected static final String FIGURE = "figure";
 138         protected static final String FIGURE_LEGEND = "figureLegend";
 139         protected static final String FIGURE_PART = "figurePart";
 140         protected static final String FIGURE_REF = "figureRef";
 141         protected static final String FIGURE_TITLE = "figureTitle";
 142         protected static final String FOOTNOTE_STRING = "footnoteString";
 143         protected static final String FREQUENCY = "frequency";
 144         protected static final String HEADING = "heading";
 145         protected static final String HABITAT = "habitat";
 146         protected static final String HABITAT_LIST = "habitatList";
 147         protected static final String IS_FREETEXT = "isFreetext";
 148         protected static final String ID = "id";
 149         protected static final String KEY = "key";
 150         protected static final String LIFE_CYCLE_PERIODS = "lifeCyclePeriods";
 151         protected static final String META_DATA = "metaData";
 152         protected static final String MODS = "mods";
 153
 154         protected static final String NOMENCLATURE = "nomenclature";
 155         protected static final String QUOTE = "quote";
 156         protected static final String RANK = "rank";
 157         protected static final String REF = "ref";
 158         protected static final String REF_NUM = "refNum";
 159         protected static final String REFERENCE = "reference";
 160         protected static final String REFERENCES = "references";
 161         protected static final String SUB_CHAR = "subChar";
 162         protected static final String TAXON = "taxon";
 163         protected static final String TAXONTITLE = "taxontitle";
 164         protected static final String TAXONTYPE = "taxontype";
 165         protected static final String TEXT_SECTION = "textSection";
 166         protected static final String TREATMENT = "treatment";
 167         protected static final String SERIALS_ABBREVIATIONS = "serialsAbbreviations";
 168         protected static final String STRING = "string";
 169         protected static final String URL = "url";
 170         protected static final String WRITER = "writer";
 171
 172         protected static final String LOCALITY = "locality";
 173
 174
 175
 176         //Nomenclature
 177         protected static final String ACCEPTED = "accepted";
 178         protected static final String ACCEPTED_NAME = "acceptedName";
 179         protected static final String ALTERNATEPUBTITLE = "alternatepubtitle";
 180         protected static final String AUTHOR = "author";
 181         protected static final String DETAILS = "details";
 182         protected static final String EDITION = "edition";
 183         protected static final String EDITORS = "editors";
 184         protected static final String HOMONYM = "homonym";
 185         protected static final String HOMOTYPES = "homotypes";
 186         protected static final String NOMENCLATURAL_NOTES = "nomenclaturalNotes";
 187         protected static final String INFRANK = "infrank";
 188         protected static final String INFRAUT = "infraut";
 189         protected static final String INFRPARAUT = "infrparaut";
 190         protected static final String ISSUE = "issue";
 191         protected static final String NAME = "name";
 192         protected static final String NAME_TYPE = "nameType";
 193         protected static final String NOM = "nom";
 194         protected static final String PAGES = "pages";
 195         protected static final String PARAUT = "paraut";
 196         protected static final String PUBFULLNAME = "pubfullname";
 197         protected static final String PUBLOCATION = "publocation";
 198         protected static final String PUBLISHER = "publisher";
 199         protected static final String PUBNAME = "pubname";
 200         protected static final String PUBTITLE = "pubtitle";
 201         protected static final String PUBTYPE = "pubtype";
 202         protected static final String REF_PART = "refPart";
 203         protected static final String SYNONYM = "synonym";
 204         protected static final String USAGE = "usage";
 205         protected static final String VOLUME = "volume";
 206         protected static final String YEAR = "year";
 207
 208
 209         //keys
 210         protected static final String COUPLET = "couplet";
 211         protected static final String IS_SPOTCHARACTERS = "isSpotcharacters";
 212         protected static final String ONLY_NUMBERED_TAXA_EXIST = "onlyNumberedTaxaExist";
 213         protected static final String EXISTS = "exists";
 214         protected static final String KEYNOTES = "keynotes";
 215         protected static final String KEY_TITLE = "keyTitle";
 216         protected static final String QUESTION = "question";
 217         protected static final String TEXT = "text";
 218         protected static final String TO_COUPLET = "toCouplet";
 219         protected static final String TO_KEY = "toKey";
 220         protected static final String TO_TAXON = "toTaxon";
 221
 222
 223         //Feature
 224         protected static final String VERNACULAR_NAMES = "vernacularNames";
 225         protected static final String VERNACULAR_NAME = "vernacularName";
 226         protected static final String TRANSLATION = "translation";
 227         protected static final String LOCAL_LANGUAGE = "localLanguage";
 228
 229
 230
 231         protected MarkupDocumentImport docImport;
 232
 233         private IEditGeoService editGeoService;
 234
 235         public MarkupImportBase(MarkupDocumentImport docImport) {
 236                 super();
 237                 this.docImport = docImport;
 238                 this.editGeoService = docImport.getEditGeoService();
 239         }
 240
 241         private Stack<QName> unhandledElements = new Stack<QName>();
 242         private Stack<QName> handledElements = new Stack<QName>();
 243
 244
 245         protected <T extends CdmBase> void  save(Collection<T> collection, MarkupImportState state) {
 246                 if (state.isCheck() || collection.isEmpty()){
 247                         return;
 248                 }
 249                 T example = collection.iterator().next();
 250                 if (example.isInstanceOf(TaxonBase.class)){
 251                         Collection<TaxonBase> typedCollection = (Collection<TaxonBase>)collection;
 252                         docImport.getTaxonService().saveOrUpdate(typedCollection);
 253                 }else if (example.isInstanceOf(Classification.class)){
 254                         Collection<Classification> typedCollection = (Collection<Classification>)collection;
 255                         docImport.getClassificationService().saveOrUpdate(typedCollection);
 256                 }else if (example.isInstanceOf(PolytomousKey.class)){
 257                         Collection<PolytomousKey> typedCollection = (Collection<PolytomousKey>)collection;
 258                         docImport.getPolytomousKeyService().saveOrUpdate(typedCollection);
 259                 }else if (example.isInstanceOf(DefinedTermBase.class)){
 260                         Collection<DefinedTermBase> typedCollection = (Collection<DefinedTermBase>)collection;
 261                         getTermService().saveOrUpdate(typedCollection);
 262                 }
 263
 264         }
 265
 266
 267         //TODO move to service layer for all IdentifiableEntities
 268         protected void save(CdmBase cdmBase, MarkupImportState state) {
 269                 if (state.isCheck()){
 270                         return;
 271                 }
 272                 cdmBase = CdmBase.deproxy(cdmBase, CdmBase.class);
 273                 if (cdmBase == null){
 274                         String message = "Tried to save a null object.";
 275                         fireWarningEvent(message, "--location ?? --", 6,1);
 276                 } else if (cdmBase.isInstanceOf(TaxonBase.class)){
 277                         docImport.getTaxonService().saveOrUpdate((TaxonBase<?>)cdmBase);
 278                 }else if (cdmBase.isInstanceOf(Classification.class)){
 279                         docImport.getClassificationService().saveOrUpdate((Classification)cdmBase);
 280                 }else if (cdmBase.isInstanceOf(PolytomousKey.class)){
 281                         docImport.getPolytomousKeyService().saveOrUpdate((PolytomousKey)cdmBase);
 282                 }else if (cdmBase.isInstanceOf(DefinedTermBase.class)){
 283                         docImport.getTermService().saveOrUpdate((DefinedTermBase<?>)cdmBase);
 284                 }else if (cdmBase.isInstanceOf(Media.class)){
 285                         docImport.getMediaService().saveOrUpdate((Media)cdmBase);
 286                 }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 287                         docImport.getOccurrenceService().saveOrUpdate((SpecimenOrObservationBase<?>)cdmBase);
 288                 }else if (cdmBase.isInstanceOf(DescriptionElementBase.class)){
 289                         docImport.getDescriptionService().saveDescriptionElement((DescriptionElementBase)cdmBase);
 290                 }else if (cdmBase.isInstanceOf(Reference.class)){
 291                         docImport.getReferenceService().saveOrUpdate((Reference<?>)cdmBase);
 292                 }else{
 293                         String message = "Unknown cdmBase type to save: " + cdmBase.getClass();
 294                         fireWarningEvent(message, "Unknown location", 8);
 295                 }
 296                 //logger.warn("Saved " +  cdmBase);
 297         }
 298
 299
 300         protected ITermService getTermService() {
 301                 return docImport.getTermService();
 302         }
 303
 304         protected IClassificationService getClassificationService() {
 305                 return docImport.getClassificationService();
 306         }
 307
 308 //*********************** Attribute methods *************************************/
 309
 310         /**
 311          * Returns a map for all attributes of an start element
 312          * @param event
 313          * @return
 314          */
 315         protected Map<String, Attribute> getAttributes(XMLEvent event) {
 316                 Map<String, Attribute> result = new HashMap<String, Attribute>();
 317                 if (!event.isStartElement()){
 318                         fireWarningEvent("Event is not an startElement. Can't check attributes", makeLocationStr(event.getLocation()), 1, 1);
 319                         return result;
 320                 }
 321                 StartElement element = event.asStartElement();
 322                 Iterator<Attribute> attributes = element.getAttributes();
 323                 while (attributes.hasNext()){
 324                         Attribute attribute = attributes.next();
 325                         //TODO namespaces
 326                         result.put(attribute.getName().getLocalPart(), attribute);
 327                 }
 328                 return result;
 329         }
 330
 331         /**
 332          * Throws an unexpected attributes event if the event has any attributes.
 333          * @param event
 334          */
 335         protected void checkNoAttributes(Map<String, Attribute> attributes, XMLEvent event) {
 336                 String[] exceptions = new String[]{};
 337                 handleUnexpectedAttributes(event.getLocation(), attributes, 1, exceptions);
 338         }
 339
 340
 341
 342         /**
 343          * Throws an unexpected attributes event if the event has any attributes.
 344          * @param event
 345          */
 346         protected void checkNoAttributes(XMLEvent event) {
 347                 String[] exceptions = new String[]{};
 348                 checkNoAttributes(event, 1, exceptions);
 349         }
 350
 351         /**
 352          * Throws an unexpected attributes event if the event has any attributes except those mentioned in "exceptions".
 353          * @param event
 354          * @param exceptions
 355          */
 356         protected void checkNoAttributes(XMLEvent event, int stackDepth, String... exceptions) {
 357                 if (! event.isStartElement()){
 358                         fireWarningEvent("Event is not an startElement. Can't check attributes", makeLocationStr(event.getLocation()), 1, 1);
 359                         return;
 360                 }
 361                 StartElement startElement = event.asStartElement();
 362                 Map<String, Attribute> attributes = getAttributes(startElement);
 363                 handleUnexpectedAttributes(startElement.getLocation(), attributes, stackDepth+1, exceptions);
 364         }
 365
 366
 367         /**
 368          * Checks if the given attribute exists and has the given value.
 369          * If yes, true is returned and the attribute is removed from the attributes map.
 370          * Otherwise false is returned.
 371          * @param attributes
 372          * @param attrName
 373          * @param value
 374          * @return <code>true</code> if attribute has given value, <code>false</code> otherwise
 375          */
 376         protected boolean checkAndRemoveAttributeValue( Map<String, Attribute> attributes, String attrName, String value) {
 377                 Attribute attr = attributes.get(attrName);
 378                 if (attr == null ||value == null ){
 379                         return false;
 380                 }else{
 381                         if (value.equals(attr.getValue())){
 382                                 attributes.remove(attrName);
 383                                 return true;
 384                         }else{
 385                                 return false;
 386                         }
 387                 }
 388         }
 389
 390
 391         /**
 392          * Returns the value of a given attribute name and removes the attribute from the attributes map.
 393          * @param attributes
 394          * @param attrName
 395          * @return
 396          */
 397         protected String getAndRemoveAttributeValue(Map<String, Attribute> attributes, String attrName) {
 398                 return getAndRemoveAttributeValue(null, attributes, attrName, false, 1);
 399         }
 400
 401         /**
 402          * Returns the value of a boolean attribute with the given name and removes the attribute from the attributes map.
 403          * Returns <code>defaultValue</code> if the attribute does not exist. ALso returns <code>defaultValue</code> and throws a warning if the
 404          * attribute has no boolean value (true, false).
 405          * @param
 406          * @param attributes the
 407          * @param attrName the name of the attribute
 408          * @param defaultValue the default value to return if attribute does not exist or can not be defined
 409          * @return
 410          */
 411         protected Boolean getAndRemoveBooleanAttributeValue(XMLEvent event, Map<String, Attribute> attributes, String attrName, Boolean defaultValue) {
 412                 String value = getAndRemoveAttributeValue(null, attributes, attrName, false, 1);
 413                 Boolean result = defaultValue;
 414                 if (value != null){
 415                         if (value.equalsIgnoreCase("true")){
 416                                 result = true;
 417                         }else if (value.equalsIgnoreCase("false")){
 418                                 result = false;
 419                         }else{
 420                                 String message = "Boolean attribute has no boolean value ('true', 'false') but '%s'";
 421                                 fireWarningEvent(String.format(message, value), makeLocationStr(event.getLocation()), 6, 1);
 422                         }
 423                 }
 424                 return result;
 425         }
 426
 427
 428         /**
 429          * Returns the value of a given attribute name and returns the attribute from the attributes map.
 430          * Fires a mandatory field is missing event if the attribute does not exist.
 431          * @param xmlEvent
 432          * @param attributes
 433          * @param attrName
 434          * @return
 435          */
 436         protected String getAndRemoveRequiredAttributeValue(XMLEvent xmlEvent, Map<String, Attribute> attributes, String attrName) {
 437                 return getAndRemoveAttributeValue(xmlEvent, attributes, attrName, true, 1);
 438         }
 439
 440         /**
 441          * Returns the value of a given attribute name and returns the attribute from the attributes map.
 442          * If required is <code>true</code> and the attribute does not exist a mandatory field is missing event is fired.
 443          * @param xmlEvent
 444          * @param attributes
 445          * @param attrName
 446          * @param isRequired
 447          * @return
 448          */
 449         private String getAndRemoveAttributeValue(XMLEvent xmlEvent, Map<String, Attribute> attributes, String attrName, boolean isRequired, int stackDepth) {
 450                 Attribute attr = attributes.get(attrName);
 451                 if (attr == null ){
 452                         if (isRequired){
 453                                 fireMandatoryElementIsMissing(xmlEvent, attrName, 8, stackDepth+1);
 454                         }
 455                         return null;
 456                 }else{
 457                         attributes.remove(attrName);
 458                         return attr.getValue();
 459                 }
 460         }
 461
 462         /**
 463          * Fires an not yet implemented event if the given attribute exists in attributes.
 464          * @param attributes
 465          * @param attrName
 466          */
 467         protected void handleNotYetImplementedAttribute(Map<String, Attribute>  attributes, String attrName) {
 468                 Attribute attr = attributes.get(attrName);
 469                 if (attr != null){
 470                         attributes.remove(attrName);
 471                         QName qName = attr.getName();
 472                         fireNotYetImplementedAttribute(attr.getLocation(), qName, 1);
 473                 }
 474         }
 475
 476         /**
 477          * Fires an unhandled attributes event, if attributes exist in attributes map not covered by the exceptions.
 478          * No event is fired if the unhandled elements stack is not empty.
 479          * @param location
 480          * @param attributes
 481          * @param exceptions
 482          */
 483         protected void handleUnexpectedAttributes(Location location,Map<String, Attribute> attributes, String... exceptions) {
 484                 handleUnexpectedAttributes(location, attributes, 1, exceptions);
 485         }
 486
 487         /**
 488          * see {@link #handleUnexpectedAttributes(Location, Map, String...)}
 489      *
 490          * @param location
 491          * @param attributes
 492          * @param stackDepth the stack trace depth
 493          * @param exceptions
 494          */
 495         private void handleUnexpectedAttributes(Location location,Map<String, Attribute> attributes, int stackDepth, String... exceptions) {
 496                 if (attributes.size() > 0){
 497                         if (this.unhandledElements.size() == 0 ){
 498                                 boolean hasUnhandledAttributes = false;
 499                                 for (String key : attributes.keySet()){
 500                                         boolean isException = false;
 501                                         for (String exception : exceptions){
 502                                                 if(key.equals(exception)){
 503                                                         isException = true;
 504                                                 }
 505                                         }
 506                                         if (!isException){
 507                                                 hasUnhandledAttributes = true;
 508                                         }
 509                                 }
 510                                 if (hasUnhandledAttributes){
 511                                         fireUnexpectedAttributes(location, attributes, stackDepth+1);
 512                                 }
 513                         }
 514                 }
 515         }
 516
 517
 518         private void fireUnexpectedAttributes(Location location, Map<String, Attribute> attributes, int stackDepth) {
 519                 String attributesString = "";
 520                 for (String key : attributes.keySet()){
 521                         Attribute attribute = attributes.get(key);
 522                         attributesString = CdmUtils.concat(",", attributesString, attribute.getName().getLocalPart() + ":" + attribute.getValue());
 523                 }
 524                 String message = "Unexpected attributes: %s";
 525                 IoProblemEvent event = makeProblemEvent(location, String.format(message, attributesString), 1 , stackDepth +1 );
 526                 fire(event);
 527         }
 528
 529
 530         protected void fireUnexpectedAttributeValue(XMLEvent parentEvent, String attrName, String attrValue) {
 531                 String message = "Unexpected attribute value %s='%s'";
 532                 message = String.format(message, attrName, attrValue);
 533                 IoProblemEvent event = makeProblemEvent(parentEvent.getLocation(), message, 1 , 1 );
 534                 fire(event);
 535         }
 536
 537         protected void handleNotYetImplementedAttributeValue(XMLEvent xmlEvent, String attrName, String attrValue) {
 538                 String message = "Attribute %s not yet implemented for value '%s'";
 539                 message = String.format(message, attrName, attrValue);
 540                 IIoEvent event = makeProblemEvent(xmlEvent.getLocation(), message, 1, 1 );
 541                 fire(event);
 542         }
 543
 544         protected void fireNotYetImplementedAttribute(Location location, QName qName, int stackDepth) {
 545                 String message = "Attribute not yet implemented: %s";
 546                 IIoEvent event = makeProblemEvent(location, String.format(message, qName.getLocalPart()), 1, stackDepth+1 );
 547                 fire(event);
 548         }
 549
 550
 551         protected void fireUnexpectedEvent(XMLEvent xmlEvent, int stackDepth) {
 552                 Location location = xmlEvent.getLocation();
 553                 String message = "Unexpected event: %s";
 554                 IIoEvent event = makeProblemEvent(location, String.format(message, xmlEvent.toString()), 2, stackDepth +1);
 555                 fire(event);
 556         }
 557
 558         protected void fireUnexpectedStartElement(Location location, StartElement startElement, int stackDepth) {
 559                 QName qName = startElement.getName();
 560                 String message = "Unexpected start element: %s";
 561                 IIoEvent event = makeProblemEvent(location, String.format(message, qName.getLocalPart()), 2, stackDepth +1);
 562                 fire(event);
 563         }
 564
 565
 566         protected void fireUnexpectedEndElement(Location location, EndElement endElement, int stackDepth) {
 567                 QName qName = endElement.getName();
 568                 String message = "Unexpected end element: %s";
 569                 IIoEvent event = makeProblemEvent(location, String.format(message, qName.getLocalPart()), 16, stackDepth+1);
 570                 fire(event);
 571         }
 572
 573         protected void fireNotYetImplementedElement(Location location, QName qName, int stackDepth) {
 574                 String message = "Element not yet implemented: %s";
 575                 IIoEvent event = makeProblemEvent(location, String.format(message, qName.getLocalPart()), 1, stackDepth+1 );
 576                 fire(event);
 577         }
 578
 579         protected void fireNotYetImplementedCharacters(Location location, Characters chars, int stackDepth) {
 580                 String message = "Characters not yet handled: %s";
 581                 IIoEvent event = makeProblemEvent(location, String.format(message, chars.getData()), 1, stackDepth+1 );
 582                 fire(event);
 583         }
 584
 585         /**
 586          * Creates a problem event.
 587          * Be aware of the right depths of the stack trace !
 588          * @param location
 589          * @param message
 590          * @param severity
 591          * @return
 592          */
 593         private IoProblemEvent makeProblemEvent(Location location, String message, int severity, int stackDepth) {
 594                 stackDepth++;
 595                 StackTraceElement[] stackTrace = new Exception().getStackTrace();
 596                 int lineNumber = stackTrace[stackDepth].getLineNumber();
 597                 String methodName = stackTrace[stackDepth].getMethodName();
 598                 String locationStr = makeLocationStr(location);
 599                 String className = stackTrace[stackDepth].getClassName();
 600                 Class<?> declaringClass;
 601                 try {
 602                         declaringClass = Class.forName(className);
 603                 } catch (ClassNotFoundException e) {
 604                         declaringClass = this.getClass();
 605                 }
 606                 IoProblemEvent event = IoProblemEvent.NewInstance(declaringClass, message,
 607                                 locationStr, lineNumber, severity, methodName);
 608                 return event;
 609         }
 610
 611         /**
 612          * Creates a string from a location
 613          * @param location
 614          * @return
 615          */
 616         protected String makeLocationStr(Location location) {
 617                 String locationStr = location == null ? " - no location - " : "l." + location.getLineNumber() + "/c."+ location.getColumnNumber();
 618                 return locationStr;
 619         }
 620
 621
 622         /**
 623          * Fires an unexpected element event if the unhandled elements stack is empty.
 624          * Otherwise adds the element to the stack.
 625          * @param event
 626          */
 627         protected void handleUnexpectedStartElement(XMLEvent event) {
 628                 handleUnexpectedStartElement(event, 1);
 629         }
 630
 631         /**
 632          * Fires an unexpected element event if the unhandled elements stack is empty.
 633          * Otherwise adds the element to the stack.
 634          * @param event
 635          */
 636         protected void handleUnexpectedStartElement(XMLEvent event, int stackDepth) {
 637                 QName qName = event.asStartElement().getName();
 638                 if (! unhandledElements.empty()){
 639                         unhandledElements.push(qName);
 640                 }else{
 641                         fireUnexpectedStartElement(event.getLocation(), event.asStartElement(), stackDepth + 1);
 642                 }
 643         }
 644
 645
 646         protected void handleUnexpectedEndElement(EndElement event) {
 647                 handleUnexpectedEndElement(event, 1);
 648         }
 649
 650         /**
 651          * Fires an unexpected element event if the event is not the last on the stack.
 652          * Otherwise removes last stack element.
 653          * @param event
 654          */
 655         protected void handleUnexpectedEndElement(EndElement event, int stackDepth) {
 656                 QName qName = event.asEndElement().getName();
 657                 if (!unhandledElements.isEmpty() && unhandledElements.peek().equals(qName)){
 658                         unhandledElements.pop();
 659                 }else{
 660                         fireUnexpectedEndElement(event.getLocation(), event.asEndElement(), stackDepth + 1);
 661                 }
 662         }
 663
 664         /**
 665          *
 666          * @param endElement
 667          */
 668         protected void popUnimplemented(EndElement endElement) {
 669                 QName qName = endElement.asEndElement().getName();
 670                 if (unhandledElements.peek().equals(qName)){
 671                         unhandledElements.pop();
 672                 }else{
 673                         String message = "End element is not last on stack: %s";
 674                         message = String.format(message, qName.getLocalPart());
 675                         IIoEvent event = makeProblemEvent(endElement.getLocation(), message, 16, 1);
 676                         fire(event);
 677                 }
 678
 679         }
 680
 681
 682         /**
 683          * Fires an unexpected element event if the unhandled element stack is empty.
 684          * @param event
 685          */
 686         protected void handleUnexpectedElement(XMLEvent event) {
 687                 if (event.isStartElement()){
 688                         handleUnexpectedStartElement(event, 2);
 689                 }else if (event.isEndElement()){
 690                         handleUnexpectedEndElement(event.asEndElement(), 2);
 691                 }else if (event.getEventType() == XMLStreamConstants.COMMENT){
 692                         //do nothing
 693                 }else if (! unhandledElements.empty()){
 694                         //do nothing
 695                 }else{
 696                         fireUnexpectedEvent(event, 1);
 697                 }
 698         }
 699
 700         /**
 701          * Fires an not yet implemented event and adds the element name to the unhandled elements stack.
 702          * @param event
 703          */
 704         protected void handleNotYetImplementedCharacters(XMLEvent event) {
 705                 Characters chars = event.asCharacters();
 706                 fireNotYetImplementedCharacters(event.getLocation(), chars, 1);
 707         }
 708
 709         /**
 710          * Fires an not yet implemented event and adds the element name to the unhandled elements stack.
 711          * @param event
 712          */
 713         protected void handleNotYetImplementedElement(XMLEvent event) {
 714                 QName qName = event.asStartElement().getName();
 715                 boolean isTopLevel = unhandledElements.isEmpty();
 716                 unhandledElements.push(qName);
 717                 if (isTopLevel){
 718                         fireNotYetImplementedElement(event.getLocation(), qName, 1);
 719                 }
 720         }
 721
 722         /**
 723          * Fires an not yet implemented event and adds the element name to the unhandled elements stack.
 724          * @param event
 725          */
 726         protected void handleIgnoreElement(XMLEvent event) {
 727                 QName qName = event.asStartElement().getName();
 728                 unhandledElements.push(qName);
 729         }
 730
 731         protected void handleAmbigousManually(MarkupImportState state,
 732                         XMLEventReader reader, StartElement startElement) {
 733                 QName qName = startElement.getName();
 734                 unhandledElements.push(qName);
 735                 fireWarningEvent(
 736                                 "Handle manually: " + qName.getLocalPart() + " is ambigous and should therefore be handled manually",
 737                                 makeLocationStr(startElement.getLocation()), 2, 2);
 738         }
 739
 740         /**
 741          * Checks if a mandatory text is not empty or null.
 742          * Returns true if text is given.
 743          * Fires an mandatory element is missing event otherwise and returns <code>null</code>.
 744          * @param text
 745          * @param parentEvent
 746          * @return
 747          */
 748         protected boolean checkMandatoryText(String text, XMLEvent parentEvent) {
 749                 if (! StringUtils.isNotBlank(text)){
 750                         fireMandatoryElementIsMissing(parentEvent, "CData", 4, 1);
 751                         return false;
 752                 }
 753                 return true;
 754         }
 755
 756         /**
 757          * Fires an mandatory element is missing event if exists is <code>false</code>.
 758          * @param hasMandatory
 759          * @param parentEvent
 760          * @param string
 761          */
 762         protected void checkMandatoryElement(boolean exists, StartElement parentEvent, String attrName) {
 763                 if (! exists){
 764                         fireMandatoryElementIsMissing(parentEvent, attrName, 5, 1);
 765                 }
 766         }
 767
 768
 769         /**
 770          * Fires an element is missing event.
 771          * @param xmlEvent
 772          * @param string
 773          * @param severity
 774          * @param stackDepth
 775          * @throws IllegalStateException if xmlEvent is not a StartElement and not an Attribute
 776          */
 777         private void fireMandatoryElementIsMissing(XMLEvent xmlEvent, String missingEventName, int severity, int stackDepth) throws IllegalStateException{
 778                 Location location = xmlEvent.getLocation();
 779                 String typeName;
 780                 QName qName;
 781                 if (xmlEvent.isAttribute()){
 782                         Attribute attribute = ((Attribute)xmlEvent);
 783                         typeName = "attribute";
 784                         qName = attribute.getName();
 785                 }else if (xmlEvent.isStartElement()){
 786                         typeName = "element";
 787                         qName = xmlEvent.asStartElement().getName();
 788                 }else{
 789                         throw new IllegalStateException("mandatory element only allowed for attributes and start tags in " + makeLocationStr(location));
 790                 }
 791                 String message = "Mandatory %s '%s' is missing in %s";
 792                 message = String.format(message, typeName , missingEventName, qName.getLocalPart());
 793                 IIoEvent event = makeProblemEvent(location, message, severity, stackDepth +1);
 794                 fire(event);
 795         }
 796
 797
 798
 799
 800         /**
 801          * Returns true if the "next" event is the ending tag for the "parent" event.
 802          * @param next end element to test, must not be null
 803          * @param parentEvent start element to test
 804          * @return true if the "next" event is the ending tag for the "parent" event.
 805          * @throws XMLStreamException
 806          */
 807         protected boolean isMyEndingElement(XMLEvent next, XMLEvent parentEvent) throws XMLStreamException {
 808                 if (! parentEvent.isStartElement()){
 809                         String message = "Parent event should be start tag";
 810                         fireWarningEvent(message, makeLocationStr(next.getLocation()), 6);
 811                         return false;
 812                 }
 813                 return isEndingElement(next, parentEvent.asStartElement().getName().getLocalPart());
 814         }
 815
 816         /**
 817          * Trims the text and removes turns all whitespaces into single empty space.
 818          * @param text
 819          * @return
 820          */
 821         protected String normalize(String text) {
 822                 text = StringUtils.trimToEmpty(text);
 823                 text = text.replaceAll("\\s+", " ");
 824                 return text;
 825         }
 826
 827
 828
 829         /**
 830          * Removes whitespaces at beginning and end and makes the first letter
 831          * a capital letter and all other letters small letters.
 832          * @param value
 833          * @return
 834          */
 835         protected String toFirstCapital(String value) {
 836                 if (StringUtils.isBlank(value)){
 837                         return value;
 838                 }else{
 839                         String result = "";
 840                         value = value.trim();
 841                         result += value.trim().substring(0,1).toUpperCase();
 842                         if (value.length()>1){
 843                                 result += value.substring(1).toLowerCase();
 844                         }
 845                         return result;
 846                 }
 847         }
 848
 849         /**
 850          * Currently not used.
 851          * @param str
 852          * @param allowedNumberOfCharacters
 853          * @param onlyFirstCapital
 854          * @return
 855          */
 856         protected boolean isAbbreviation(String str, int allowedNumberOfCharacters, boolean onlyFirstCapital){
 857                 if (isBlank(str)){
 858                         return false;
 859                 }
 860                 str = str.trim();
 861                 if (! str.endsWith(".")){
 862                         return false;
 863                 }
 864                 str = str.substring(0, str.length() -1);
 865                 if (str.length() > allowedNumberOfCharacters){
 866                         return false;
 867                 }
 868                 final String re = "^\\p{javaUpperCase}\\p{javaLowerCase}*$";
 869                 if (str.matches(re)){
 870                         return true;
 871                 }else{
 872                         return false;
 873                 }
 874         }
 875
 876         /**
 877          * Checks if <code>abbrev</code> is the short form for the genus name (strGenusName).
 878          * Usually this is the case if <code>abbrev</code> is the first letter (optional with ".")
 879          * of strGenusName. But in older floras it may also be the first 2 or 3 letters (optional with dot).
 880          * However, we allow only a maximum of 2 letters to be anambigous. In cases with 3 letters better
 881          * change the original markup data.
 882          * @param single
 883          * @param strGenusName
 884          * @return
 885          */
 886         protected boolean isGenusAbbrev(String abbrev, String strGenusName) {
 887                 if (! abbrev.matches("[A-Z][a-z]?\\.?")) {
 888                         return false;
 889                 }else if (abbrev.length() == 0 || strGenusName == null || strGenusName.length() == 0){
 890                         return false;
 891                 }else{
 892                         abbrev = abbrev.replace(".", "");
 893                         return strGenusName.startsWith(abbrev);
 894 //                      boolean result = true;
 895 //                      for (int i = 0 ; i < abbrev.length(); i++){
 896 //                              result &= ( abbrev.charAt(i) == strGenusName.charAt(i));
 897 //                      }
 898 //                      return result;
 899                 }
 900         }
 901
 902
 903         /**
 904          * Checks if all words in the given string start with a capital letter but do not have any further capital letter.
 905          * @param word the string to be checekd. Usually should be a single word.
 906          * @return true if the above is the case, false otherwise
 907          */
 908         protected boolean isFirstCapitalWord(String word) {
 909                 if (WordUtils.capitalizeFully(word).equals(word)){
 910                         return true;
 911                 }else if (WordUtils.capitalizeFully(word,new char[]{'-'}).equals(word)){
 912                         //for words like Le-Testui (which is a species epithet)
 913                         return true;
 914                 }else{
 915                         return false;
 916                 }
 917         }
 918
 919
 920         /**
 921          * Read next event. Ignore whitespace events.
 922          * @param reader
 923          * @return
 924          * @throws XMLStreamException
 925          */
 926         protected XMLEvent readNoWhitespace(XMLEventReader reader) throws XMLStreamException {
 927                 XMLEvent event = reader.nextEvent();
 928                 while (!unhandledElements.isEmpty()){
 929                         if (event.isStartElement()){
 930                                 handleNotYetImplementedElement(event);
 931                         }else if (event.isEndElement()){
 932                                 popUnimplemented(event.asEndElement());
 933                         }
 934                         event = reader.nextEvent();
 935                 }
 936                 while (event.isCharacters() && event.asCharacters().isWhiteSpace()){
 937                         event = reader.nextEvent();
 938                 }
 939                 return event;
 940         }
 941
 942         /**
 943          * Returns the REQUIRED "class" attribute for a given event and checks that it is the only attribute.
 944          * @param parentEvent
 945          * @return
 946          */
 947         protected String getClassOnlyAttribute(XMLEvent parentEvent) {
 948                 return getClassOnlyAttribute(parentEvent, true);
 949         }
 950
 951
 952         /**
 953          * Returns the "class" attribute for a given event and checks that it is the only attribute.
 954          * @param parentEvent
 955          * @return
 956          */
 957         protected String getClassOnlyAttribute(XMLEvent parentEvent, boolean required) {
 958                 return getOnlyAttribute(parentEvent, CLASS, required);
 959         }
 960
 961         /**
 962          * Returns the value for the only attribute for a given event and checks that it is the only attribute.
 963          * @param parentEvent
 964          * @return
 965          */
 966         protected String getOnlyAttribute(XMLEvent parentEvent, String attrName, boolean required) {
 967                 Map<String, Attribute> attributes = getAttributes(parentEvent);
 968                 String classValue =getAndRemoveAttributeValue(parentEvent, attributes, attrName, required, 1);
 969                 checkNoAttributes(attributes, parentEvent);
 970                 return classValue;
 971         }
 972
 973
 974         protected void fireWarningEvent(String message, String locationStr, Integer severity, Integer depth) {
 975                 docImport.fireWarningEvent(message, locationStr, severity, depth);
 976         }
 977
 978         protected void fireWarningEvent(String message, XMLEvent event, Integer severity) {
 979                 docImport.fireWarningEvent(message, makeLocationStr(event.getLocation()), severity, 1);
 980         }
 981
 982         protected void fireSchemaConflictEventExpectedStartTag(String elName, XMLEventReader reader) throws XMLStreamException {
 983                 docImport.fireSchemaConflictEventExpectedStartTag(elName, reader);
 984         }
 985
 986
 987         protected void fireWarningEvent(String message, String locationStr, int severity) {
 988                 docImport.fireWarningEvent(message, locationStr, severity, 1);
 989         }
 990
 991         protected void fire(IIoEvent event) {
 992                 docImport.fire(event);
 993         }
 994
 995         protected boolean isNotBlank(String str){
 996                 return StringUtils.isNotBlank(str);
 997         }
 998
 999         protected boolean isBlank(String str){
1000                 return StringUtils.isBlank(str);
1001         }
1002
1003         protected TaxonDescription getTaxonDescription(Taxon taxon, Reference<?> ref, boolean isImageGallery, boolean createNewIfNotExists) {
1004                 return docImport.getTaxonDescription(taxon, isImageGallery, createNewIfNotExists);
1005         }
1006
1007
1008         /**
1009          * Returns the default language defined in the state. If no default language is defined in the state,
1010          * the CDM default language is returned.
1011          * @param state
1012          * @return
1013          */
1014         protected Language getDefaultLanguage(MarkupImportState state) {
1015                 Language result = state.getDefaultLanguage();
1016                 if (result == null){
1017                         result = Language.DEFAULT();
1018                 }
1019                 return result;
1020         }
1021
1022
1023 //*********************** FROM XML IMPORT BASE ****************************************
1024         protected boolean isEndingElement(XMLEvent event, String elName) throws XMLStreamException {
1025                 return docImport.isEndingElement(event, elName);
1026         }
1027
1028         protected boolean isStartingElement(XMLEvent event, String elName) throws XMLStreamException {
1029                 return docImport.isStartingElement(event, elName);
1030         }
1031
1032
1033         protected void fillMissingEpithetsForTaxa(Taxon parentTaxon, Taxon childTaxon) {
1034                 docImport.fillMissingEpithetsForTaxa(parentTaxon, childTaxon);
1035         }
1036
1037         protected Feature getFeature(MarkupImportState state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<Feature> voc){
1038                 return docImport.getFeature(state, uuid, label, text, labelAbbrev, voc);
1039         }
1040
1041         protected ExtensionType getExtensionType(MarkupImportState state, UUID uuid, String label, String text, String labelAbbrev){
1042                 return docImport.getExtensionType(state, uuid, label, text, labelAbbrev);
1043         }
1044
1045         protected DefinedTerm getIdentifierType(MarkupImportState state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<DefinedTerm> voc){
1046                 return docImport.getIdentifierType(state, uuid, label, text, labelAbbrev, voc);
1047         }
1048
1049         protected AnnotationType getAnnotationType(MarkupImportState state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<AnnotationType> voc){
1050                 return docImport.getAnnotationType(state, uuid, label, text, labelAbbrev, voc);
1051         }
1052
1053         protected MarkerType getMarkerType(MarkupImportState state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<MarkerType> voc){
1054                 return docImport.getMarkerType(state, uuid, label, text, labelAbbrev, voc);
1055         }
1056
1057         protected NamedAreaLevel getNamedAreaLevel(MarkupImportState state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<NamedAreaLevel> voc){
1058                 return docImport.getNamedAreaLevel(state, uuid, label, text, labelAbbrev, voc);
1059         }
1060
1061         protected NamedArea getNamedArea(MarkupImportState state, UUID uuid, String label, String text, String labelAbbrev, NamedAreaType areaType, NamedAreaLevel level, TermVocabulary voc, TermMatchMode matchMode){
1062                 return docImport.getNamedArea(state, uuid, label, text, labelAbbrev, areaType, level, voc, matchMode);
1063         }
1064
1065         protected Language getLanguage(MarkupImportState state, UUID uuid, String label, String text, String labelAbbrev, TermVocabulary<?> voc){
1066                 return docImport.getLanguage(state, uuid, label, text, labelAbbrev, voc);
1067         }
1068
1069 // *************************************** Concrete methods **********************************************/
1070
1071
1072         /**
1073          * @param state
1074          * @param classValue
1075          * @param byAbbrev
1076          * @return
1077          */
1078         protected Rank makeRank(MarkupImportState state, String value, boolean byAbbrev) {
1079                 Rank rank = null;
1080                 if (StringUtils.isBlank(value)) {
1081                         return null;
1082                 }
1083                 try {
1084                         boolean useUnknown = true;
1085                         NomenclaturalCode nc = makeNomenclaturalCode(state);
1086                         if (value.equals(GENUS_ABBREVIATION)){
1087                                 rank = Rank.GENUS();
1088                         }else if (byAbbrev) {
1089                                 rank = Rank.getRankByIdInVoc(value, nc, useUnknown);
1090                         } else {
1091                                 rank = Rank.getRankByEnglishName(value, nc, useUnknown);
1092                         }
1093                         if (rank.equals(Rank.UNKNOWN_RANK())) {
1094                                 rank = null;
1095                         }
1096                         if (rank == null && "sous-genre".equalsIgnoreCase(value)){
1097                                 rank = Rank.SUBGENUS();
1098                         }
1099                 } catch (UnknownCdmTypeException e) {
1100                         // doNothing
1101                 }
1102                 return rank;
1103         }
1104
1105
1106
1107         protected TeamOrPersonBase<?> createAuthor(String authorTitle) {
1108                 // TODO atomize and also use by name creation
1109                 TeamOrPersonBase<?> result = Team.NewTitledInstance(authorTitle, authorTitle);
1110                 return result;
1111         }
1112
1113         protected String getAndRemoveMapKey(Map<String, String> map, String key) {
1114                 String result = map.get(key);
1115                 map.remove(key);
1116                 if (result != null) {
1117                         result = normalize(result);
1118                 }
1119                 return StringUtils.stripToNull(result);
1120         }
1121
1122
1123         /**
1124          * Creates a {@link NonViralName} object depending on the defined {@link NomenclaturalCode}
1125          * and the given parameters.
1126          * @param state
1127          * @param rank
1128          * @return
1129          */
1130         protected NonViralName<?> createNameByCode(MarkupImportState state, Rank rank) {
1131                 NonViralName<?> name;
1132                 NomenclaturalCode nc = makeNomenclaturalCode(state);
1133                 name = (NonViralName<?>) nc.getNewTaxonNameInstance(rank);
1134                 return name;
1135         }
1136
1137         protected void handleFullName(MarkupImportState state, XMLEventReader reader,
1138                         NonViralName<?> name, XMLEvent next) throws XMLStreamException {
1139                 String fullNameStr;
1140                 Map<String, Attribute> attrs = getAttributes(next);
1141                 String rankStr = getAndRemoveRequiredAttributeValue(next,
1142                                 attrs, "rank");
1143                 Rank rank = makeRank(state, rankStr, false);
1144                 name.setRank(rank);
1145                 if (rank == null) {
1146                         String message = "Rank was computed as null. This must not be.";
1147                         fireWarningEvent(message, next, 6);
1148                         name.setRank(Rank.UNKNOWN_RANK());
1149                 }
1150                 if (!attrs.isEmpty()) {
1151                         handleUnexpectedAttributes(next.getLocation(), attrs);
1152                 }
1153 //              next = readNoWhitespace(reader);
1154                 fullNameStr = getCData(state, reader, next, false);
1155                 NonViralNameParserImpl.NewInstance().parseFullName(name, fullNameStr, rank, false);
1156 //              name.setTitleCache(fullNameStr, true);
1157         }
1158
1159
1160         /**
1161          * Returns the {@link NomenclaturalCode} for this import. Default is {@link NomenclaturalCode#ICBN} if
1162          * no code is defined.
1163          * @param state
1164          * @return
1165          */
1166         protected NomenclaturalCode makeNomenclaturalCode(MarkupImportState state) {
1167                 NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
1168                 if (nc == null) {
1169                         nc = NomenclaturalCode.ICNAFP; // default;
1170                 }
1171                 return nc;
1172         }
1173
1174
1175         /**
1176          * @param state
1177          * @param levelString
1178          * @param next
1179          * @return
1180          */
1181         protected NamedAreaLevel makeNamedAreaLevel(MarkupImportState state, String levelString, XMLEvent next) {
1182                 NamedAreaLevel level;
1183                 try {
1184                         level = state.getTransformer().getNamedAreaLevelByKey(levelString);
1185                         if (level == null) {
1186                                 UUID levelUuid = state.getTransformer().getNamedAreaLevelUuid(levelString);
1187                                 if (levelUuid == null) {
1188                                         String message = "Unknown distribution locality class (named area level): %s. Create new level instead.";
1189                                         message = String.format(message, levelString);
1190                                         fireWarningEvent(message, next, 6);
1191                                 }
1192                                 level = getNamedAreaLevel(state, levelUuid, levelString, levelString, levelString, null);
1193                         }
1194                 } catch (UndefinedTransformerMethodException e) {
1195                         throw new RuntimeException(e);
1196                 }
1197                 return level;
1198         }
1199
1200
1201         /**
1202          * @param state
1203          * @param areaName
1204          * @param level
1205          * @return
1206          */
1207         protected NamedArea makeArea(MarkupImportState state, String areaName, NamedAreaLevel level) {
1208
1209                 //TODO FM vocabulary
1210                 TermVocabulary<NamedArea> voc = null;
1211                 NamedAreaType areaType = null;
1212
1213                 NamedArea area = null;
1214                 try {
1215                         area = state.getTransformer().getNamedAreaByKey(areaName);
1216                 } catch (UndefinedTransformerMethodException e) {
1217                         throw new RuntimeException(e);
1218                 }
1219                 if (area == null){
1220                         boolean isNewInState = false;
1221                         UUID uuid = state.getAreaUuid(areaName);
1222                         if (uuid == null){
1223                                 isNewInState = true;
1224
1225
1226                                 try {
1227                                         uuid = state.getTransformer().getNamedAreaUuid(areaName);
1228                                 } catch (UndefinedTransformerMethodException e) {
1229                                         throw new RuntimeException(e);
1230                                 }
1231                         }
1232
1233                         CdmImportBase.TermMatchMode matchMode = CdmImportBase.TermMatchMode.UUID_LABEL;
1234                         area = getNamedArea(state, uuid, areaName, areaName, areaName, areaType, level, voc, matchMode);
1235                         if (isNewInState){
1236                                 state.putAreaUuid(areaName, area.getUuid());
1237
1238                                 //TODO just for testing -> make generic and move to better place
1239                                 String geoServiceLayer="vmap0_as_bnd_political_boundary_a";
1240                                 String layerFieldName ="nam";
1241
1242                                 if ("Bangka".equals(areaName)){
1243                                         String areaValue = "PULAU BANGKA#SUMATERA SELATAN";
1244                                         GeoServiceArea geoServiceArea = new GeoServiceArea();
1245                                         geoServiceArea.add(geoServiceLayer, layerFieldName, areaValue);
1246                                         this.editGeoService.setMapping(area, geoServiceArea);
1247 //                                      save(area, state);
1248                                 }
1249                                 if ("Luzon".equals(areaName)){
1250                                         GeoServiceArea geoServiceArea = new GeoServiceArea();
1251
1252                                         List<String> list = Arrays.asList("HERMANA MAYOR ISLAND#CENTRAL LUZON",
1253                                                         "HERMANA MENOR ISLAND#CENTRAL LUZON",
1254                                                         "CENTRAL LUZON");
1255                                         for (String areaValue : list){
1256                                                 geoServiceArea.add(geoServiceLayer, layerFieldName, areaValue);
1257                                         }
1258
1259                                         this.editGeoService.setMapping(area, geoServiceArea);
1260 //                                      save(area, state);
1261                                 }
1262                                 if ("Mindanao".equals(areaName)){
1263                                         GeoServiceArea geoServiceArea = new GeoServiceArea();
1264
1265                                         List<String> list = Arrays.asList("NORTHERN MINDANAO",
1266                                                         "SOUTHERN MINDANAO",
1267                                                         "WESTERN MINDANAO");
1268                                         //TODO to be continued
1269                                         for (String areaValue : list){
1270                                                 geoServiceArea.add(geoServiceLayer, layerFieldName, areaValue);
1271                                         }
1272
1273                                         this.editGeoService.setMapping(area, geoServiceArea);
1274 //                                      save(area, state);
1275                                 }
1276                                 if ("Palawan".equals(areaName)){
1277                                         GeoServiceArea geoServiceArea = new GeoServiceArea();
1278
1279                                         List<String> list = Arrays.asList("PALAWAN#SOUTHERN TAGALOG");
1280                                         for (String areaValue : list){
1281                                                 geoServiceArea.add(geoServiceLayer, layerFieldName, areaValue);
1282                                         }
1283
1284                                         this.editGeoService.setMapping(area, geoServiceArea);
1285 //                                      save(area, state);
1286                                 }
1287
1288                         }
1289                 }
1290                 return area;
1291         }
1292
1293
1294
1295         /**
1296          * Reads character data. Any element other than character data or the ending
1297          * tag will fire an unexpected element event.
1298      *
1299          * @see #getCData(MarkupImportState, XMLEventReader, XMLEvent, boolean)
1300          * @param state
1301          * @param reader
1302          * @param next
1303          * @return
1304          * @throws XMLStreamException
1305          */
1306         protected String getCData(MarkupImportState state, XMLEventReader reader, XMLEvent next) throws XMLStreamException {
1307                 return getCData(state, reader, next, true);
1308         }
1309
1310         /**
1311          * Reads character data. Any element other than character data or the ending
1312          * tag will fire an unexpected element event.
1313          *
1314          * @param state
1315          * @param reader
1316          * @param next
1317          * @param inlineMarkup map for inline markup, this is used for e.g. the locality markup within a subheading
1318          * The map will be filled by the markup element name as key. The value may be a String, a CdmBase or any other object.
1319          * If null any markup text will be neglected but a warning will be fired if they exist.
1320          * @param removeInlineMarkupText if true the markedup text will be removed from the returned String
1321          * @param checkAttributes
1322          * @return
1323          * @throws XMLStreamException
1324          */
1325         protected String getCData(MarkupImportState state, XMLEventReader reader, XMLEvent parent, /*Map<String, Object> inlineMarkup, *boolean removeInlineMarkupText,*/ boolean checkAttributes) throws XMLStreamException {
1326                 if (checkAttributes){
1327                         checkNoAttributes(parent);
1328                 }
1329
1330                 String text = "";
1331                 while (reader.hasNext()) {
1332                         XMLEvent next = readNoWhitespace(reader);
1333                         if (isMyEndingElement(next, parent)) {
1334                                 return text;
1335                         } else if (next.isCharacters()) {
1336                                 text += next.asCharacters().getData();
1337                         } else if (isStartingElement(next, FOOTNOTE_REF)){
1338                                 handleNotYetImplementedElement(next);
1339 //                      } else if (isStartingElement(next, LOCALITY)){
1340 //                              handleCDataLocality(state, reader, parent);
1341                         } else {
1342                                 handleUnexpectedElement(next);
1343                         }
1344                 }
1345                 throw new IllegalStateException("Event has no closing tag");
1346
1347         }
1348
1349 //      private void handleCDataLocality(MarkupImportState state, XMLEventReader reader, XMLEvent parent) {
1350 //              checkAndRemoveAttributeValue(attributes, attrName, value)
1351 //
1352 //      }
1353
1354
1355
1356         /**
1357          * For it returns a pure CData annotation string. This behaviour may change in future. More complex annotations
1358          * should be handled differently.
1359          * @param state
1360          * @param reader
1361          * @param parentEvent
1362          * @return
1363          * @throws XMLStreamException
1364          */
1365         protected String handleSimpleAnnotation(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1366                 String annotation = getCData(state, reader, parentEvent);
1367                 return annotation;
1368         }
1369
1370         /**
1371          * True if text is single "." oder "," or ";" or ":"
1372          * @param text
1373          * @return
1374          */
1375         protected boolean isPunctuation(String text) {
1376                 return text == null ? false : text.trim().matches("^[\\.,;:]$");
1377         }
1378
1379
1380         /**
1381          * Text indicating that type information is following but no information about the type of the type
1382          * @param text
1383          * @return
1384          */
1385         protected boolean charIsSimpleType(String text) {
1386                 return text.matches("(?i)Type:");
1387         }
1388
1389         protected String getXmlTag(XMLEvent event) {
1390                 String result;
1391                 if (event.isStartElement()) {
1392                         result = "<" + event.asStartElement().getName().getLocalPart()
1393                                         + ">";
1394                 } else if (event.isEndElement()) {
1395                         result = "</" + event.asEndElement().getName().getLocalPart() + ">";
1396                 } else {
1397                         String message = "Only start or end elements are allowed as Html tags";
1398                         throw new IllegalStateException(message);
1399                 }
1400                 return result;
1401         }
1402
1403         protected WriterDataHolder handleWriter(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1404                 String text = "";
1405                 checkNoAttributes(parentEvent);
1406                 WriterDataHolder dataHolder = new WriterDataHolder();
1407                 List<FootnoteDataHolder> footnotes = new ArrayList<FootnoteDataHolder>();
1408
1409                 // TODO handle attributes
1410                 while (reader.hasNext()) {
1411                         XMLEvent next = readNoWhitespace(reader);
1412                         if (isMyEndingElement(next, parentEvent)) {
1413                                 text = CdmUtils.removeBrackets(text);
1414                                 if (checkMandatoryText(text, parentEvent)) {
1415                                         text = normalize(text);
1416                                         dataHolder.writer = text;
1417                                         dataHolder.footnotes = footnotes;
1418
1419                                         // Extension
1420                                         UUID uuidWriterExtension = MarkupTransformer.uuidWriterExtension;
1421                                         ExtensionType writerExtensionType =
1422                                                         this.getExtensionType(state, uuidWriterExtension,"Writer", "writer", "writer");
1423                                         Extension extension = Extension.NewInstance();
1424                                         extension.setType(writerExtensionType);
1425                                         extension.setValue(text);
1426                                         dataHolder.extension = extension;
1427
1428                                         // Annotation
1429                                         UUID uuidWriterAnnotation = MarkupTransformer.uuidWriterAnnotation;
1430                                         AnnotationType writerAnnotationType = this.getAnnotationType(state, uuidWriterAnnotation, "Writer", "writer", "writer", null);
1431                                         Annotation annotation = Annotation.NewInstance(text, writerAnnotationType, getDefaultLanguage(state));
1432                                         dataHolder.annotation = annotation;
1433
1434                                         return dataHolder;
1435                                 } else {
1436                                         return null;
1437                                 }
1438                         } else if (isStartingElement(next, FOOTNOTE_REF)) {
1439                                 FootnoteDataHolder footNote = handleFootnoteRef(state, reader, next);
1440                                 if (footNote.isRef()) {
1441                                         footnotes.add(footNote);
1442                                 } else {
1443                                         logger.warn("Non ref footnotes not yet impelemnted");
1444                                 }
1445                         } else if (next.isCharacters()) {
1446                                 text += next.asCharacters().getData();
1447
1448                         } else {
1449                                 handleUnexpectedElement(next);
1450                                 state.setUnsuccessfull();
1451                         }
1452                 }
1453                 throw new IllegalStateException("<writer> has no end tag");
1454         }
1455
1456
1457         protected void registerFootnotes(MarkupImportState state, AnnotatableEntity entity, List<FootnoteDataHolder> footnotes) {
1458                 for (FootnoteDataHolder footNote : footnotes) {
1459                         registerFootnoteDemand(state, entity, footNote);
1460                 }
1461         }
1462
1463
1464         private void registerFootnoteDemand(MarkupImportState state, AnnotatableEntity entity, FootnoteDataHolder footnote) {
1465                 FootnoteDataHolder existingFootnote = state.getFootnote(footnote.ref);
1466                 if (existingFootnote != null) {
1467                         attachFootnote(state, entity, existingFootnote);
1468                 } else {
1469                         Set<AnnotatableEntity> demands = state.getFootnoteDemands(footnote.ref);
1470                         if (demands == null) {
1471                                 demands = new HashSet<AnnotatableEntity>();
1472                                 state.putFootnoteDemands(footnote.ref, demands);
1473                         }
1474                         demands.add(entity);
1475                 }
1476         }
1477
1478
1479         protected void attachFootnote(MarkupImportState state, AnnotatableEntity entity, FootnoteDataHolder footnote) {
1480                 AnnotationType annotationType = this.getAnnotationType(state, MarkupTransformer.uuidFootnote, "Footnote", "An e-flora footnote", "fn", null);
1481                 Annotation annotation = Annotation.NewInstance(footnote.string, annotationType, getDefaultLanguage(state));
1482                 // TODO transient objects
1483                 entity.addAnnotation(annotation);
1484                 save(entity, state);
1485         }
1486
1487
1488         protected void attachFigure(MarkupImportState state, XMLEvent next, AnnotatableEntity entity, Media figure) {
1489                 // IdentifiableEntity<?> toSave;
1490                 if (entity.isInstanceOf(TextData.class)) {
1491                         TextData deb = CdmBase.deproxy(entity, TextData.class);
1492                         deb.addMedia(figure);
1493                         // toSave = ((TaxonDescription)deb.getInDescription()).getTaxon();
1494                 } else if (entity.isInstanceOf(SpecimenOrObservationBase.class)) {
1495                         String message = "figures for specimen should be handled as Textdata";
1496                         fireWarningEvent(message, next, 4);
1497                         // toSave = ime;
1498                 } else if (entity.isInstanceOf(IdentifiableMediaEntity.class)) {
1499                         IdentifiableMediaEntity<?> ime = CdmBase.deproxy(entity, IdentifiableMediaEntity.class);
1500                         ime.addMedia(figure);
1501                         // toSave = ime;
1502                 } else {
1503                         String message = "Unsupported entity to attach media: %s";
1504                         message = String.format(message, entity.getClass().getName());
1505                         // toSave = null;
1506                 }
1507                 save(entity, state);
1508         }
1509
1510
1511         protected void registerGivenFootnote(MarkupImportState state, FootnoteDataHolder footnote) {
1512                 state.registerFootnote(footnote);
1513                 Set<AnnotatableEntity> demands = state.getFootnoteDemands(footnote.id);
1514                 if (demands != null) {
1515                         for (AnnotatableEntity entity : demands) {
1516                                 attachFootnote(state, entity, footnote);
1517                         }
1518                 }
1519         }
1520
1521
1522         protected FootnoteDataHolder handleFootnote(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent,
1523                         MarkupSpecimenImport specimenImport, MarkupNomenclatureImport nomenclatureImport) throws XMLStreamException {
1524                 FootnoteDataHolder result = new FootnoteDataHolder();
1525                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1526                 result.id = getAndRemoveAttributeValue(attributes, ID);
1527                 // result.ref = getAndRemoveAttributeValue(attributes, REF);
1528                 checkNoAttributes(attributes, parentEvent);
1529
1530                 while (reader.hasNext()) {
1531                         XMLEvent next = readNoWhitespace(reader);
1532                         if (isStartingElement(next, FOOTNOTE_STRING)) {
1533                                 String string = handleFootnoteString(state, reader, next, specimenImport, nomenclatureImport);
1534                                 result.string = string;
1535                         } else if (isMyEndingElement(next, parentEvent)) {
1536                                 return result;
1537                         } else {
1538                                 fireUnexpectedEvent(next, 0);
1539                         }
1540                 }
1541                 return result;
1542         }
1543
1544
1545         protected Media handleFigure(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent,
1546                         MarkupSpecimenImport specimenImport, MarkupNomenclatureImport nomenclatureImport) throws XMLStreamException {
1547                 // FigureDataHolder result = new FigureDataHolder();
1548
1549                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1550                 String id = getAndRemoveAttributeValue(attributes, ID);
1551                 String type = getAndRemoveAttributeValue(attributes, TYPE);
1552                 String urlAttr = getAndRemoveAttributeValue(attributes, URL);
1553                 checkNoAttributes(attributes, parentEvent);
1554
1555                 String urlString = null;
1556                 String legendString = null;
1557                 String titleString = null;
1558                 String numString = null;
1559                 String text = null;
1560                 if (isNotBlank(urlAttr)){
1561                         urlString = CdmUtils.Nz(state.getBaseMediaUrl()) + urlAttr;
1562                 }
1563                 while (reader.hasNext()) {
1564                         XMLEvent next = readNoWhitespace(reader);
1565                         if (isMyEndingElement(next, parentEvent)) {
1566                                 if (isNotBlank(text)){
1567                                         fireWarningEvent("Text not yet handled for figures: " + text, next, 4);
1568                                 }
1569                                 Media media = makeFigure(state, id, type, urlString, legendString, titleString, numString, next);
1570                                 return media;
1571                         } else if (isStartingElement(next, FIGURE_LEGEND)) {
1572                                 // TODO same as figure string ?
1573                                 legendString = handleFootnoteString(state, reader, next, specimenImport, nomenclatureImport);
1574                         } else if (isStartingElement(next, FIGURE_TITLE)) {
1575                                 titleString = getCData(state, reader, next);
1576                         } else if (isStartingElement(next, URL)) {
1577                                 String localUrl = getCData(state, reader, next);
1578                                 String url = CdmUtils.Nz(state.getBaseMediaUrl()) + localUrl;
1579                                 if (isBlank(urlString)){
1580                                         urlString = url;
1581                                 }
1582                                 if (! url.equals(urlString)){
1583                                         String message = "URL attribute and URL element differ. Attribute: %s, Element: %s";
1584                                         fireWarningEvent(String.format(message, urlString, url), next, 2);
1585                                 }
1586                         } else if (isStartingElement(next, NUM)) {
1587                                 numString = getCData(state, reader, next);
1588                         } else if (next.isCharacters()) {
1589                                 text += CdmUtils.concat("", text, next.asCharacters().getData());
1590                         } else {
1591                                 fireUnexpectedEvent(next, 0);
1592                         }
1593                 }
1594                 throw new IllegalStateException("<figure> has no end tag");
1595         }
1596
1597
1598         /**
1599          * @param state
1600          * @param id
1601          * @param type
1602          * @param urlString
1603          * @param legendString
1604          * @param titleString
1605          * @param numString
1606          * @param next
1607          */
1608         private Media makeFigure(MarkupImportState state, String id, String type, String urlString,
1609                         String legendString, String titleString, String numString, XMLEvent next) {
1610                 Media media = null;
1611 //              boolean isFigure = false;  //no difference between figure and media since v3.3
1612                 try {
1613                         //TODO maybe everything is a figure as it is all taken from a book
1614                         if ("lineart".equals(type)) {
1615 //                              isFigure = true;
1616 //                              media = Figure.NewInstance(url.toURI(), null, null,     null);
1617                         } else if (type == null || "photo".equals(type)
1618                                         || "signature".equals(type)
1619                                         || "others".equals(type)) {
1620                                 //TODO
1621                         } else {
1622                                 String message = "Unknown figure type '%s'";
1623                                 message = String.format(message, type);
1624                                 fireWarningEvent(message, next, 2);
1625                         }
1626                         media = docImport.getImageMedia(urlString, docImport.getReadMediaData());
1627
1628                         if (media != null){
1629                                 // title
1630                                 if (StringUtils.isNotBlank(titleString)) {
1631                                         media.putTitle(getDefaultLanguage(state), titleString);
1632                                 }
1633                                 // legend
1634                                 if (StringUtils.isNotBlank(legendString)) {
1635                                         media.putDescription(getDefaultLanguage(state), legendString);
1636                                 }
1637                                 if (StringUtils.isNotBlank(numString)) {
1638                                         // TODO use concrete source (e.g. DAPHNIPHYLLACEAE in FM
1639                                         // vol.13)
1640                                         Reference<?> citation = state.getConfig().getSourceReference();
1641                                         media.addSource(OriginalSourceType.Import, numString, "num", citation, null);
1642                                         // TODO name used in source if available
1643                                 }
1644                                 // TODO which citation
1645                                 if (StringUtils.isNotBlank(id)) {
1646                                         media.addSource(OriginalSourceType.Import, id, null, state.getConfig().getSourceReference(), null);
1647                                 } else {
1648                                         String message = "Figure id should never be empty or null";
1649                                         fireWarningEvent(message, next, 6);
1650                                 }
1651
1652                                 // text
1653                                 // do nothing
1654                                 registerGivenFigure(state, next, id, media);
1655
1656                         }else{
1657                                 String message = "No media found: ";
1658                                 fireWarningEvent(message, next, 4);
1659                         }
1660                 } catch (MalformedURLException e) {
1661                         String message = "Media uri has incorrect syntax: %s";
1662                         message = String.format(message, urlString);
1663                         fireWarningEvent(message, next, 4);
1664 //              } catch (URISyntaxException e) {
1665 //                      String message = "Media uri has incorrect syntax: %s";
1666 //                      message = String.format(message, urlString);
1667 //                      fireWarningEvent(message, next, 4);
1668                 }
1669
1670                 return media;
1671         }
1672
1673
1674         private void registerGivenFigure(MarkupImportState state, XMLEvent next, String id, Media figure) {
1675                 state.registerFigure(id, figure);
1676                 Set<AnnotatableEntity> demands = state.getFigureDemands(id);
1677                 if (demands != null) {
1678                         for (AnnotatableEntity entity : demands) {
1679                                 attachFigure(state, next, entity, figure);
1680                         }
1681                 }
1682                 save(figure, state);
1683         }
1684
1685
1686         private FootnoteDataHolder handleFootnoteRef(MarkupImportState state,
1687                         XMLEventReader reader, XMLEvent parentEvent)
1688                         throws XMLStreamException {
1689                 FootnoteDataHolder result = new FootnoteDataHolder();
1690                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1691                 result.ref = getAndRemoveAttributeValue(attributes, REF);
1692                 checkNoAttributes(attributes, parentEvent);
1693
1694                 // text is not handled, needed only for debugging purposes
1695                 String text = "";
1696                 while (reader.hasNext()) {
1697                         XMLEvent next = readNoWhitespace(reader);
1698                         // if (isStartingElement(next, FOOTNOTE_STRING)){
1699                         // String string = handleFootnoteString(state, reader, next);
1700                         // result.string = string;
1701                         // }else
1702                         if (isMyEndingElement(next, parentEvent)) {
1703                                 if (StringUtils.isNotBlank(text)){
1704                                         fireWarningEvent("text is not empty but not handled during import", parentEvent, 4);
1705                                 }
1706                                 return result;
1707                         } else if (next.isCharacters() && unhandledElements.isEmpty()) {
1708                                 text += next.asCharacters().getData();
1709                         } else if (isStartingElement(next, NUM)) {
1710                                 //ignore numbering of footnotes as they are numbered differently in the CDM
1711                                 handleIgnoreElement(next);
1712                         } else {
1713                                 handleUnexpectedElement(next);
1714                         }
1715                 }
1716                 return result;
1717         }
1718
1719
1720
1721         private String handleFootnoteString(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent, MarkupSpecimenImport specimenImport, MarkupNomenclatureImport nomenclatureImport) throws XMLStreamException {
1722                 boolean isTextMode = true;
1723                 String text = "";
1724                 while (reader.hasNext()) {
1725                         XMLEvent next = readNoWhitespace(reader);
1726                         if (isMyEndingElement(next, parentEvent)) {
1727                                 return text;
1728                         } else if (next.isEndElement()) {
1729                                 if (isEndingElement(next, FULL_NAME)) {
1730                                         popUnimplemented(next.asEndElement());
1731                                 } else if (isEndingElement(next, BR)) {
1732                                         isTextMode = true;
1733                                 } else if (isHtml(next)) {
1734                                         text += getXmlTag(next);
1735                                 } else {
1736                                         handleUnexpectedEndElement(next.asEndElement());
1737                                 }
1738                         } else if (next.isStartElement()) {
1739                                 if (isStartingElement(next, FULL_NAME)) {
1740                                         handleNotYetImplementedElement(next);
1741                                 } else if (isStartingElement(next, GATHERING)) {
1742                                         text += specimenImport.handleInLineGathering(state, reader, next);
1743                                 } else if (isStartingElement(next, REFERENCES)) {
1744                                         text += " " + handleInLineReferences(state, reader, next, nomenclatureImport)+ " ";
1745                                 } else if (isStartingElement(next, BR)) {
1746                                         text += "<br/>";
1747                                         isTextMode = false;
1748                                 } else if (isStartingElement(next, NOMENCLATURE)) {
1749                                         handleNotYetImplementedElement(next);
1750                                 } else if (isHtml(next)) {
1751                                         text += getXmlTag(next);
1752                                 } else {
1753                                         handleUnexpectedStartElement(next.asStartElement());
1754                                 }
1755                         } else if (next.isCharacters()) {
1756                                 if (!isTextMode) {
1757                                         String message = "footnoteString is not in text mode";
1758                                         fireWarningEvent(message, next, 6);
1759                                 } else {
1760                                         text += next.asCharacters().getData().trim();
1761                                         // getCData(state, reader, next); does not work as we have inner tags like <references>
1762                                 }
1763                         } else {
1764                                 handleUnexpectedEndElement(next.asEndElement());
1765                         }
1766                 }
1767                 throw new IllegalStateException("<footnoteString> has no closing tag");
1768
1769         }
1770
1771         private static final List<String> htmlList = Arrays.asList("sub", "sup",
1772                         "ol", "ul", "li", "i", "b", "table", "br","tr","td");
1773
1774         protected boolean isHtml(XMLEvent event) {
1775                 if (event.isStartElement()) {
1776                         String tag = event.asStartElement().getName().getLocalPart();
1777                         return htmlList.contains(tag);
1778                 } else if (event.isEndElement()) {
1779                         String tag = event.asEndElement().getName().getLocalPart();
1780                         return htmlList.contains(tag);
1781                 } else {
1782                         return false;
1783                 }
1784
1785         }
1786
1787
1788         private String handleInLineReferences(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent, MarkupNomenclatureImport nomenclatureImport) throws XMLStreamException {
1789                 checkNoAttributes(parentEvent);
1790
1791                 boolean hasReference = false;
1792                 String text = "";
1793                 while (reader.hasNext()) {
1794                         XMLEvent next = readNoWhitespace(reader);
1795                         if (isMyEndingElement(next, parentEvent)) {
1796                                 checkMandatoryElement(hasReference, parentEvent.asStartElement(), REFERENCE);
1797                                 return text;
1798                         } else if (isStartingElement(next, REFERENCE)) {
1799                                 text += handleInLineReference(state, reader, next, nomenclatureImport);
1800                                 hasReference = true;
1801                         } else {
1802                                 handleUnexpectedElement(next);
1803                         }
1804                 }
1805                 throw new IllegalStateException("<References> has no closing tag");
1806         }
1807
1808         private String handleInLineReference(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent, MarkupNomenclatureImport nomenclatureImport)throws XMLStreamException {
1809                 Reference<?> reference = nomenclatureImport.handleReference(state, reader, parentEvent);
1810                 String result = "<cdm:ref uuid='%s'>%s</ref>";
1811                 result = String.format(result, reference.getUuid(), reference.getTitleCache());
1812                 save(reference, state);
1813                 return result;
1814         }
1815
1816
1817         /**
1818          * Handle string
1819          * @param state
1820          * @param reader
1821          * @param parentEvent
1822          * @param feature only needed for distributionLocalities
1823          * @return
1824          * @throws XMLStreamException
1825          */
1826         protected Map<String, String> handleString(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent, Feature feature)throws XMLStreamException {
1827                 // attributes
1828                 String classValue = getClassOnlyAttribute(parentEvent, false);
1829                 if (StringUtils.isNotBlank(classValue)) {
1830                         String message = "class attribute for <string> not yet implemented";
1831                         fireWarningEvent(message, parentEvent, 2);
1832                 }
1833
1834                 // subheadings
1835                 Map<String, String> subHeadingMap = new HashMap<String, String>();
1836                 String currentSubheading = null;
1837
1838                 boolean isTextMode = true;
1839                 String text = "";
1840                 while (reader.hasNext()) {
1841                         XMLEvent next = readNoWhitespace(reader);
1842                         if (isMyEndingElement(next, parentEvent)) {
1843                                 putCurrentSubheading(subHeadingMap, currentSubheading, text);
1844                                 return subHeadingMap;
1845                         } else if (isStartingElement(next, BR)) {
1846                                 text += "<br/>";
1847                                 isTextMode = false;
1848                         } else if (isEndingElement(next, BR)) {
1849                                 isTextMode = true;
1850                         } else if (isHtml(next)) {
1851                                 text += getXmlTag(next);
1852                         } else if (isStartingElement(next, SUB_HEADING)) {
1853                                 text = putCurrentSubheading(subHeadingMap,currentSubheading, text);
1854                                 // TODO footnotes
1855                                 currentSubheading = getCData(state, reader, next).trim();
1856                         } else if (isStartingElement(next, DISTRIBUTION_LOCALITY)) {
1857                                 if (feature != null && !feature.equals(Feature.DISTRIBUTION())) {
1858                                         String message = "Distribution locality only allowed for feature of type 'distribution'";
1859                                         fireWarningEvent(message, next, 4);
1860                                 }
1861                                 text += handleDistributionLocality(state, reader, next);
1862                         } else if (next.isCharacters()) {
1863                                 if (! isTextMode) {
1864                                         String message = "String is not in text mode";
1865                                         fireWarningEvent(message, next, 6);
1866                                 } else {
1867                                         text += next.asCharacters().getData();
1868                                 }
1869                         } else if (isStartingElement(next, HEADING)) {
1870                                 //TODO
1871                                 handleNotYetImplementedElement(next);
1872                         } else if (isStartingElement(next, VERNACULAR_NAMES)) {
1873                                 //TODO
1874                                 handleNotYetImplementedElement(next);
1875                         } else if (isStartingElement(next, QUOTE)) {
1876                                 //TODO
1877                                 handleNotYetImplementedElement(next);
1878                         } else if (isStartingElement(next, DEDICATION)) {
1879                                 //TODO
1880                                 handleNotYetImplementedElement(next);
1881                         } else if (isStartingElement(next, TAXONTYPE)) {
1882                                 //TODO
1883                                 handleNotYetImplementedElement(next);
1884                         } else if (isStartingElement(next, FULL_NAME)) {
1885                                 //TODO
1886                                 handleNotYetImplementedElement(next);
1887                         }else if (isStartingElement(next, REFERENCES)) {
1888                                 //TODO
1889                                 handleNotYetImplementedElement(next);
1890                         } else if (isStartingElement(next, GATHERING)) {
1891                                 //TODO
1892                                 handleNotYetImplementedElement(next);
1893                         } else if (isStartingElement(next, ANNOTATION)) {
1894                                 //TODO  //TODO test handleSimpleAnnotation
1895                                 handleNotYetImplementedElement(next);
1896                         } else if (isStartingElement(next, HABITAT)) {
1897                                 //TODO
1898                                 handleNotYetImplementedElement(next);
1899                         } else if (isStartingElement(next, FIGURE_REF)) {
1900                                 //TODO
1901                                 handleNotYetImplementedElement(next);
1902                         } else if (isStartingElement(next, FIGURE)) {
1903                                 //TODO
1904                                 handleNotYetImplementedElement(next);
1905                         } else if (isStartingElement(next, FOOTNOTE_REF)) {
1906                                 //TODO
1907                                 handleNotYetImplementedElement(next);
1908                         } else if (isStartingElement(next, FOOTNOTE)) {
1909                                 //TODO
1910                                 handleNotYetImplementedElement(next);
1911                         } else if (isStartingElement(next, WRITER)) {
1912                                 //TODO
1913                                 handleNotYetImplementedElement(next);
1914                         } else if (isStartingElement(next, DATES)) {
1915                                 //TODO
1916                                 handleNotYetImplementedElement(next);
1917                         } else {
1918                                 handleUnexpectedElement(next);
1919                         }
1920                 }
1921                 throw new IllegalStateException("<String> has no closing tag");
1922         }
1923
1924
1925         /**
1926          * @param subHeadingMap
1927          * @param currentSubheading
1928          * @param text
1929          * @return
1930          */
1931         private String putCurrentSubheading(Map<String, String> subHeadingMap, String currentSubheading, String text) {
1932                 if (StringUtils.isNotBlank(text)) {
1933                         text = removeStartingMinus(text);
1934                         subHeadingMap.put(currentSubheading, text.trim());
1935                 }
1936                 return "";
1937         }
1938
1939         private String removeStartingMinus(String string) {
1940                 string = replaceStart(string, "-");
1941                 string = replaceStart(string, "\u002d");
1942                 string = replaceStart(string, "\u2013");
1943                 string = replaceStart(string, "\u2014");
1944                 string = replaceStart(string, "--");
1945                 return string;
1946         }
1947
1948
1949         /**
1950          * @param value
1951          * @param replacementString
1952          */
1953         private String replaceStart(String value, String replacementString) {
1954                 if (value.startsWith(replacementString) ){
1955                         value = value.substring(replacementString.length()).trim();
1956                 }
1957                 while (value.startsWith("-") || value.startsWith("\u2014") ){
1958                         value = value.substring("-".length()).trim();
1959                 }
1960                 return value;
1961         }
1962
1963
1964         private String handleDistributionLocality(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent)throws XMLStreamException {
1965                 Map<String, Attribute> attributes = getAttributes(parentEvent);
1966                 String classValue = getAndRemoveRequiredAttributeValue(parentEvent, attributes, CLASS);
1967                 String statusValue =getAndRemoveAttributeValue(attributes, STATUS);
1968                 String frequencyValue =getAndRemoveAttributeValue(attributes, FREQUENCY);
1969
1970
1971                 Taxon taxon = state.getCurrentTaxon();
1972                 // TODO which ref to take?
1973                 Reference<?> ref = state.getConfig().getSourceReference();
1974
1975                 String text = "";
1976                 while (reader.hasNext()) {
1977                         XMLEvent next = readNoWhitespace(reader);
1978                         if (isMyEndingElement(next, parentEvent)) {
1979                                 if (StringUtils.isNotBlank(text)) {
1980                                         String label = CdmUtils.removeTrailingDot(normalize(text));
1981                                         TaxonDescription description = getTaxonDescription(taxon, ref, false, true);
1982                                         NamedAreaLevel level = makeNamedAreaLevel(state,classValue, next);
1983
1984                                         //status
1985                                         PresenceAbsenceTerm status = null;
1986                                         if (isNotBlank(statusValue)){
1987                                                 try {
1988                                                         status = state.getTransformer().getPresenceTermByKey(statusValue);
1989                                                         if (status == null){
1990                                                                 //TODO
1991                                                                 String message = "The presence/absence status '%s' could not be transformed to an CDM status";
1992                                                                 fireWarningEvent(String.format(message, statusValue), next, 4);
1993                                                         }
1994                                                 } catch (UndefinedTransformerMethodException e) {
1995                                                         throw new RuntimeException(e);
1996                                                 }
1997                                         }else{
1998                                                 status = PresenceAbsenceTerm.PRESENT();
1999                                         }
2000                                         //frequency
2001                                         if (isNotBlank(frequencyValue)){
2002                                                 String message = "The frequency attribute is currently not yet available in CDM";
2003                                                 fireWarningEvent(message, parentEvent, 6);
2004                                         }
2005
2006                                         NamedArea higherArea = null;
2007                                         List<NamedArea> areas = new ArrayList<NamedArea>();
2008
2009                                         String patSingleArea = "([^,\\(]{3,})";
2010                                         String patSeparator = "(,|\\sand\\s)";
2011                                         String hierarchiePattern = String.format("%s\\((%s(%s%s)*)\\)",patSingleArea, patSingleArea, patSeparator, patSingleArea);
2012                                         Pattern patHierarchie = Pattern.compile(hierarchiePattern, Pattern.CASE_INSENSITIVE);
2013                                         Matcher matcher = patHierarchie.matcher(label);
2014                                         if (matcher.matches()){
2015                                                 String higherAreaStr = matcher.group(1).trim();
2016                                                 higherArea =  makeArea(state, higherAreaStr, level);
2017                                                 String[] innerAreas = matcher.group(2).split(patSeparator);
2018                                                 for (String innerArea : innerAreas){
2019                                                         if (isNotBlank(innerArea)){
2020                                                                 NamedArea singleArea = makeArea(state, innerArea.trim(), level);
2021                                                                 areas.add(singleArea);
2022                                                                 NamedArea partOf = singleArea.getPartOf();
2023 //                                                              if (partOf == null){
2024 //                                                                      singleArea.setPartOf(higherArea);
2025 //                                                              }
2026                                                         }
2027                                                 }
2028                                         }else{
2029                                                 NamedArea singleArea = makeArea(state, label, level);
2030                                                 areas.add(singleArea);
2031                                         }
2032
2033                                         for (NamedArea area : areas){
2034                                                 //create distribution
2035                                                 Distribution distribution = Distribution.NewInstance(area,status);
2036                                                 description.addElement(distribution);
2037                                         }
2038                                 } else {
2039                                         String message = "Empty distribution locality";
2040                                         fireWarningEvent(message, next, 4);
2041                                 }
2042                                 return text;
2043                         } else if (isStartingElement(next, COORDINATES)) {
2044                                 //TODO
2045                                 handleNotYetImplementedElement(next);
2046                         } else if (isEndingElement(next, COORDINATES)) {
2047                                 //TODO
2048                                 popUnimplemented(next.asEndElement());
2049                         } else if (next.isCharacters()) {
2050                                 text += next.asCharacters().getData();
2051                         } else {
2052                                 handleUnexpectedElement(next);
2053                         }
2054                 }
2055                 throw new IllegalStateException("<DistributionLocality> has no closing tag");
2056         }
2057
2058
2059 //********************************************** OLD *************************************
2060
2061 //      protected boolean testAdditionalElements(Element parentElement, List<String> excludeList){
2062 //              boolean result = true;
2063 //              List<Element> list = parentElement.getChildren();
2064 //              for (Element element : list){
2065 //                      if (! excludeList.contains(element.getName())){
2066 //                              logger.warn("Unknown element (" + element.getName() + ") in parent element (" + parentElement.getName() + ")");
2067 //                              result = false;
2068 //                      }
2069 //              }
2070 //              return result;
2071 //      }
2072 //
2073 //
2074 //      protected <T extends IdentifiableEntity> T makeReferenceType(Element element, Class<? extends T> clazz, MapWrapper<? extends T> objectMap, ResultWrapper<Boolean> success){
2075 //              T result = null;
2076 //              String linkType = element.getAttributeValue("linkType");
2077 //              String ref = element.getAttributeValue("ref");
2078 //              if(ref == null && linkType == null){
2079 //                      result = getInstance(clazz);
2080 //                      if (result != null){
2081 //                              String title = element.getTextNormalize();
2082 //                              result.setTitleCache(title, true);
2083 //                      }
2084 //              }else if (linkType == null || linkType.equals("local")){
2085 //                      //TODO
2086 //                      result = objectMap.get(ref);
2087 //                      if (result == null){
2088 //                              logger.warn("Object (ref = " + ref + ")could not be found in WrapperMap");
2089 //                      }
2090 //              }else if(linkType.equals("external")){
2091 //                      logger.warn("External link types not yet implemented");
2092 //              }else if(linkType.equals("other")){
2093 //                      logger.warn("Other link types not yet implemented");
2094 //              }else{
2095 //                      logger.warn("Unknown link type or missing ref");
2096 //              }
2097 //              if (result == null){
2098 //                      success.setValue(false);
2099 //              }
2100 //              return result;
2101 //      }
2102 //
2103 //
2104 //      protected Reference makeAccordingTo(Element elAccordingTo, MapWrapper<Reference> referenceMap, ResultWrapper<Boolean> success){
2105 //              Reference result = null;
2106 //              if (elAccordingTo != null){
2107 //                      String childName = "AccordingToDetailed";
2108 //                      boolean obligatory = false;
2109 //                      Element elAccordingToDetailed = XmlHelp.getSingleChildElement(success, elAccordingTo, childName, elAccordingTo.getNamespace(), obligatory);
2110 //
2111 //                      childName = "Simple";
2112 //                      obligatory = true;
2113 //                      Element elSimple = XmlHelp.getSingleChildElement(success, elAccordingTo, childName, elAccordingTo.getNamespace(), obligatory);
2114 //
2115 //                      if (elAccordingToDetailed != null){
2116 //                              result = makeAccordingToDetailed(elAccordingToDetailed, referenceMap, success);
2117 //                      }else{
2118 //                              result = ReferenceFactory.newGeneric();
2119 //                              String title = elSimple.getTextNormalize();
2120 //                              result.setTitleCache(title, true);
2121 //                      }
2122 //              }
2123 //              return result;
2124 //      }
2125 //
2126 //
2127 //      private Reference makeAccordingToDetailed(Element elAccordingToDetailed, MapWrapper<Reference> referenceMap, ResultWrapper<Boolean> success){
2128 //              Reference result = null;
2129 //              Namespace tcsNamespace = elAccordingToDetailed.getNamespace();
2130 //              if (elAccordingToDetailed != null){
2131 //                      //AuthorTeam
2132 //                      String childName = "AuthorTeam";
2133 //                      boolean obligatory = false;
2134 //                      Element elAuthorTeam = XmlHelp.getSingleChildElement(success, elAccordingToDetailed, childName, tcsNamespace, obligatory);
2135 //                      makeAccordingToAuthorTeam(elAuthorTeam, success);
2136 //
2137 //                      //PublishedIn
2138 //                      childName = "PublishedIn";
2139 //                      obligatory = false;
2140 //                      Element elPublishedIn = XmlHelp.getSingleChildElement(success, elAccordingToDetailed, childName, tcsNamespace, obligatory);
2141 //                      result = makeReferenceType(elPublishedIn, Reference.class, referenceMap, success);
2142 //
2143 //                      //MicroReference
2144 //                      childName = "MicroReference";
2145 //                      obligatory = false;
2146 //                      Element elMicroReference = XmlHelp.getSingleChildElement(success, elAccordingToDetailed, childName, tcsNamespace, obligatory);
2147 //                      String microReference = elMicroReference.getTextNormalize();
2148 //                      if (CdmUtils.Nz(microReference).equals("")){
2149 //                              //TODO
2150 //                              logger.warn("MicroReference not yet implemented for AccordingToDetailed");
2151 //                      }
2152 //              }
2153 //              return result;
2154 //      }
2155 //
2156 //      private Team makeAccordingToAuthorTeam(Element elAuthorTeam, ResultWrapper<Boolean> succes){
2157 //              Team result = null;
2158 //              if (elAuthorTeam != null){
2159 //                      //TODO
2160 //                      logger.warn("AuthorTeam not yet implemented for AccordingToDetailed");
2161 //              }
2162 //              return result;
2163 //      }
2164
2165
2166
2167 }