# import modules & set up logging
import gensim, logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

sentences = [['first', 'sentence'], ['second', 'sentence']]
# train word2vec on the two sentences
model = gensim.models.Word2Vec(sentences, min_count=1)

from gensim.test.utils import common_texts, get_tmpfile
from gensim.models import  Word2Vec
from gensim.models import KeyedVectors

path = get_tmpfile("word2vec.model")
path2 = get_tmpfile("wordvectors.kv")

model = Word2Vec(common_texts, size=100, window=5, min_count=1, workers=4)
model.save("word2vec.model")
model.wv.save(path2)

vector = model.wv['computer']

sentences = LineSentence(datapath('lee_background.cor'))
for sentence in sentences
    pass

from gensim.models.word2vec import Word2Vec 

sentences = [['A1'，'A2']，[]，[]，....] 
model= Word2Vec()
model.build_vocab(sentences)
model.train(sentences，total_examples = model.corpus_count，epochs = model.iter)

保存：
model.save('/tmp/MyModel')
＃model.save_word2vec_format('/tmp/mymodel.txt',binary = False)
＃model.save_word2vec_format('/tmp/mymodel.bin.gz',binary = True)

追加加训练：
model = gensim.models.Word2Vec.load('/tmp/mymodel')
model.train(more_sentences)

加载：
model = gensim.models.Word2Vec.load('/tmp/mymodel')
#model = gensim.models.KeyedVectors.load_word2vec_format('/tmp/vectors.txt',binary = False)
#model = gensim.models.KeyedVectors。load_word2vec_format('/tmp/vectors.bin',binary = True)
##使用gzipped / bz2输入也可以，不需要解压：
#model = gensim.models.KeyedVectors.load_word2vec_format('/tmp/vectors.bin.gz',binary=True)

print(model ['man'])
print(type(model ['man']))
# 输出：
[0.14116223 0.05663395 0.01500377 -0.03592452 ...] 
numpy.ndarray

model.most_similar(['男人'])
# 输出：
[（'女'，0.7664012908935547），
 （ '男孩'，0.6824870109558105），
 （ '青少年'，0.6586930155754089），
 （ '女孩'，0.5921714305877686），
 （ '强盗'，0.5585119128227234），
 （ '男'，0.5489763021469116），
 （'人'，0.5420035719871521），
 （ '人'，0.5342026352882385），
 （ '绅士'，0.5337990522384644），
 （ '摩托车手'，0.5336882472038269）]

# word2vec一个很大的亮点：支持词语的加减运算（实际中可能只有少数例子比较符合）
model.most_similar(positive = ['woman','king'],negative = ['man'],topn = 2)
# 输出
[（'王后'，0.7118192911148071）,（'君主'，0.6189675331115723）]
model.similarity('女人', '男人')

0.7664012234410319

# 当出现某个词语不在这个训练集合中的时候，会报错！
list1 = ['我'，'走'，'我'，'学校'] 
list2 = ['我'，'去'，'家'] 
list_sim1 = model.n_similarity（list1，list2）
print（list_sim1）
# 输出
0.772446878519

model.doesnt_match('breakfast cereal dinner lunch'.split())
# 输出
‘谷类’

# 1. 对于给定的语料库，首先要进行分词
def segment_file(read_path, write_path):
    f2 = open(write_path, 'a', encoding='utf-8')
    # 1.1 判断路径是否存在
    if os.path.exists(read_path):
        with open(read_path, 'r', encoding='utf-8') as f:
            # f.readlines() 读取全部内容
            text = f.readlines()
            for line in text:
                line.replace('\t', '').replace('\n', '').replace(' ', '')
                seg_text = jieba.cut(line, cut_all=False)
                # print(" ".join(seg_text))
                # with open(write_path, 'a', encoding='utf-8') as f2:
                f2.write(" ".join(seg_text), )
                # print(" ".join(seg_text))
            f2.close()

# 2. 使用Word2vec模型
def train(file_path):
    logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO)
    # 加载语料
    sentences = word2vec.Text8Corpus(file_path)
    # 训练skip-gram模型， 默认windows=5
    model = word2vec.Word2Vec(sentences, size=200)
    print(model)

2018-12-10 19:34:24,044:INFO:collecting all words and their counts
2018-12-10 19:34:24,047:INFO:PROGRESS: at sentence #0, processed 0 words, keeping 0 word types
2018-12-10 19:34:24,057:INFO:collected 949 word types from a corpus of 21696 raw words and 3 sentences
2018-12-10 19:34:24,058:INFO:Loading a fresh vocabulary
2018-12-10 19:34:24,059:INFO:effective_min_count=5 retains 476 unique words (50% of original 949, drops 473)
2018-12-10 19:34:24,059:INFO:effective_min_count=5 leaves 19804 word corpus (91% of original 21696, drops 1892)
2018-12-10 19:34:24,061:INFO:deleting the raw counts dictionary of 949 items
2018-12-10 19:34:24,062:INFO:sample=0.001 downsamples 67 most-common words
2018-12-10 19:34:24,062:INFO:downsampling leaves estimated 11305 word corpus (57.1% of prior 19804)
2018-12-10 19:34:24,085:INFO:estimated required memory for 476 words and 200 dimensions: 999600 bytes
2018-12-10 19:34:24,086:INFO:resetting layer weights
2018-12-10 19:34:24,109:INFO:training model with 3 workers on 476 vocabulary and 200 features, using sg=0 hs=0 sample=0.001 negative=5 window=5
2018-12-10 19:34:24,155:INFO:worker thread finished; awaiting finish of 2 more threads
2018-12-10 19:34:24,171:INFO:worker thread finished; awaiting finish of 1 more threads
2018-12-10 19:34:24,172:INFO:worker thread finished; awaiting finish of 0 more threads
2018-12-10 19:34:24,173:INFO:EPOCH - 1 : training on 21696 raw words (11304 effective words) took 0.0s, 243500 effective words/s
2018-12-10 19:34:24,201:INFO:worker thread finished; awaiting finish of 2 more threads
2018-12-10 19:34:24,209:INFO:worker thread finished; awaiting finish of 1 more threads
2018-12-10 19:34:24,217:INFO:worker thread finished; awaiting finish of 0 more threads
2018-12-10 19:34:24,218:INFO:EPOCH - 2 : training on 21696 raw words (11360 effective words) took 0.0s, 436565 effective words/s
2018-12-10 19:34:24,247:INFO:worker thread finished; awaiting finish of 2 more threads
2018-12-10 19:34:24,250:INFO:worker thread finished; awaiting finish of 1 more threads
2018-12-10 19:34:24,255:INFO:worker thread finished; awaiting finish of 0 more threads
2018-12-10 19:34:24,255:INFO:EPOCH - 3 : training on 21696 raw words (11220 effective words) took 0.0s, 320207 effective words/s
2018-12-10 19:34:24,279:INFO:worker thread finished; awaiting finish of 2 more threads
2018-12-10 19:34:24,286:INFO:worker thread finished; awaiting finish of 1 more threads
2018-12-10 19:34:24,289:INFO:worker thread finished; awaiting finish of 0 more threads
2018-12-10 19:34:24,289:INFO:EPOCH - 4 : training on 21696 raw words (11265 effective words) took 0.0s, 349448 effective words/s
2018-12-10 19:34:24,308:INFO:worker thread finished; awaiting finish of 2 more threads
2018-12-10 19:34:24,315:INFO:worker thread finished; awaiting finish of 1 more threads
2018-12-10 19:34:24,320:INFO:worker thread finished; awaiting finish of 0 more threads
2018-12-10 19:34:24,320:INFO:EPOCH - 5 : training on 21696 raw words (11353 effective words) took 0.0s, 391712 effective words/s
2018-12-10 19:34:24,320:INFO:training on a 108480 raw words (56502 effective words) took 0.2s, 267862 effective words/s
2018-12-10 19:34:24,320:WARNING:under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay
Word2Vec(vocab=476, size=200, alpha=0.025)

# 3. 计算两个词之间的相似度
def calc_similarity(my_model):
    try:
        y1 = my_model.similarity(u"交通", u"交通事故")
    except KeyError:
        y1 = 0
    print("交通，交通事故的相似度为：", y1)

交通，交通事故的相似度为： 0.9967415

def find_Correspondence(my_model):
    print("交通事故，赔偿-")
    y = my_model.most_similar([u'交通事故', u'赔偿'], [u'书'], topn=3)
    for item in y:
        print(item[0], item[1])

# 5. 计算两个词的相关词列表
def find_twowords(my_model):
    y = my_model.most_similar(u"交通", topn=20)
    print("和交通相关的词有：")
    for item in y:
        print(item[0], item[1])

# 6. 寻找不合群的词
def find_different(my_model):
    y = my_model.doesnt_match(u"交通 肇事 罪名".split())
    print("不合群的词：", y)

E:\YanJiuSheng-download\2a\anaconda\envs\tensorflow\lib\site-packages\gensim\utils.py:1212: UserWarning: detected Windows; aliasing chunkize to chunkize_serial
  warnings.warn("detected Windows; aliasing chunkize to chunkize_serial")