实验使用mitie举行实体抽取,先纪录一下,后续改为java版本的。

  1. import mitie
    model_file='/home/test/rasa_nlu_chi/data/total_word_feature_extractor_zh.dat'
    extractor = mitie.total_word_feature_extractor(model_file)
    feats = extractor.get_feature_vector("我")
    print ("First 5 features of word 'home'", feats[0:])

  2. import sys, os
    from mitie import *
    sample = ner_training_instance(["I", "am", "looking", "for", "some", "cheap", "Mexican", "food", "."])

sample.add_entity(xrange(5,6), "pricerange")
sample.add_entity(xrange(6,7), "cuisine")

sample2 = ner_training_instance(["show", "me", "indian", "restaurants", "in", "the", "centre", "."])
sample2.add_entity(xrange(2,3), "cuisine")
sample2.add_entity(xrange(6,7), "area")

trainer = ner_trainer("/home/test/rasa_nlu_chi/data/total_word_feature_extractor_zh.dat")

trainer.add(sample)
trainer.add(sample2)

trainer.num_threads = 4

ner = trainer.train()

ner.save_to_disk("new_ner_model.dat")

tokens = ["I", "want", "expensive", "korean", "food"]
entities = ner.extract_entities(tokens)

for e in entities:
range = e[0]
tag = e[1]
entity_text = " ".join(tokens[i] for i in range)
print(" " + tag + ": " + entity_text)

  1. from mitie import *
    ner = mitie.named_entity_extractor("/home/test/rasa_nlu_chi/models/default/model_20200522-171449/entity_extractor.dat")
    tokens = ["两居室", "我的", "三居室"]
    print("Tokenized input:", tokens)
    model_file='/home/test/rasa_nlu_chi/data/total_word_feature_extractor_zh.dat'
    extractor = mitie.total_word_feature_extractor(model_file)
    entities = ner.extract_entities(tokens,extractor)
    print("\nEntities found:", entities)
    print("\nNumber of entities detected:", len(entities))
    for e in entities:
    range = e[0]
    tag = e[1]
    entity_text = " ".join(tokens[i] for i in range)
    print(" " + tag + ": " + entity_text)