一,从keras数据集imdb中加载影评数据

1,加载数据
imdb = keras.datasets.imdb
vocab_size = 10000
index_from = 3
(train_data, train_labels), (test_data, test_labels) = imdb.load_data(
    num_words = vocab_size, index_from = index_from)
2,获取id对应文字字典
word_index = imdb.get_word_index()
print(len(word_index))
# print(word_index)
word_index = {k:(v+3) for k, v in word_index.items()}
3,向字典中加入特殊字符
word_index['<PAD>'] = 0
word_index['<START>'] = 1
word_index['<UNK>'] = 2
word_index['<END>'] = 3

reverse_word_index = dict(
    [(value, key) for key, value in word_index.items()])
4,翻译 id -> 文字
def decode_review(text_ids):
    return ' '.join(
        [reverse_word_index.get(word_id, "<UNK>") for word_id in text_ids])

decode_review(train_data[0])

二,对数据做padding操作

max_length = 500

train_data = keras.preprocessing.sequence.pad_sequences(
    train_data, # list of list
    value = word_index['<PAD>'],
    padding = 'post', # post, pre
    maxlen = max_length)

test_data = keras.preprocessing.sequence.pad_sequences(
    test_data, # list of list
    value = word_index['<PAD>'],
    padding = 'post', # post, pre
    maxlen = max_length)

print(train_data[0])

三,构建模型

1,构建单层单向的LSTM模型(single_rnn_model)
# embedding_dim = 16
# batch_size = 512

# model = keras.models.Sequential([
#     # 1. define matrix: [vocab_size, embedding_dim]
#     # 2. [1,2,3,4..], max_length * embedding_dim
#     # 3. batch_size * max_length * embedding_dim
#     keras.layers.Embedding(vocab_size, embedding_dim,
#                            input_length = max_length),
#     keras.layers.LSTM(units = 64, return_sequences = False),
#     keras.layers.Dense(64, activation = 'relu'),
#     keras.layers.Dense(1, activation='sigmoid'),
# ])

# model.summary()
# model.compile(optimizer = 'adam',
#                          loss = 'binary_crossentropy',
#                          metrics = ['accuracy'])
2,构建单层单向的LSTM模型(double_rnn_model)
embedding_dim = 16
batch_size = 512

model = keras.models.Sequential([
    # 1. define matrix: [vocab_size, embedding_dim]
    # 2. [1,2,3,4..], max_length * embedding_dim
    # 3. batch_size * max_length * embedding_dim
    keras.layers.Embedding(vocab_size, embedding_dim,
                           input_length = max_length),
    keras.layers.Bidirectional(
        keras.layers.LSTM(
            units = 64, return_sequences = True)),
    keras.layers.Bidirectional(
        keras.layers.LSTM(
            units = 64, return_sequences = False)),
    keras.layers.Dense(64, activation = 'relu'),
    keras.layers.Dense(1, activation='sigmoid'),
])

model.summary()
model.compile(optimizer = 'adam',
              loss = 'binary_crossentropy',
              metrics = ['accuracy'])

四,训练模型

history_single_rnn = model.fit(
    train_data, train_labels,
    epochs = 30,
    batch_size = batch_size,
    validation_split = 0.2)

五,打印训练图表

def plot_learning_curves(history, label, epochs, min_value, max_value):
    data = {}
    data[label] = history.history[label]
    data['val_'+label] = history.history['val_'+label]
    pd.DataFrame(data).plot(figsize=(8, 5))
    plt.grid(True)
    plt.axis([0, epochs, min_value, max_value])
    plt.show()
    
plot_learning_curves(history_single_rnn, 'accuracy', 30, 0, 1)
plot_learning_curves(history_single_rnn, 'loss', 30, 0, 1)

六,打印测试集结果

res_test = model.evaluate(
    test_data, test_labels,
    batch_size = batch_size,
    verbose = 0)

print(res_test)

七,总结代码

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Mar  4 10:52:02 2021

@author: nijiahui
"""

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras

print(tf.__version__)
print(sys.version_info)
for module in mpl, np, pd, sklearn, tf, keras:
    print(module.__name__, module.__version__)
    
# 一,处理数据
# 1,加载数据
imdb = keras.datasets.imdb
vocab_size = 10000
index_from = 3
(train_data, train_labels), (test_data, test_labels) = imdb.load_data(
    num_words = vocab_size, index_from = index_from)

# 2,获取id对应文字字典
word_index = imdb.get_word_index()
print(len(word_index))
# print(word_index)
word_index = {k:(v+3) for k, v in word_index.items()}

# 3,向字典中加入特殊字符
word_index['<PAD>'] = 0
word_index['<START>'] = 1
word_index['<UNK>'] = 2
word_index['<END>'] = 3

reverse_word_index = dict(
    [(value, key) for key, value in word_index.items()])

# 4,翻译id -> 文字
def decode_review(text_ids):
    return ' '.join(
        [reverse_word_index.get(word_id, "<UNK>") for word_id in text_ids])

decode_review(train_data[0])


# 二,对数据做padding操作
max_length = 500

train_data = keras.preprocessing.sequence.pad_sequences(
    train_data, # list of list
    value = word_index['<PAD>'],
    padding = 'post', # post, pre
    maxlen = max_length)

test_data = keras.preprocessing.sequence.pad_sequences(
    test_data, # list of list
    value = word_index['<PAD>'],
    padding = 'post', # post, pre
    maxlen = max_length)

print(train_data[0])



# 三,构建模型
# # 1,构建单层单向的LSTM模型(single_rnn_model)
# embedding_dim = 16
# batch_size = 512

# model = keras.models.Sequential([
#     # 1. define matrix: [vocab_size, embedding_dim]
#     # 2. [1,2,3,4..], max_length * embedding_dim
#     # 3. batch_size * max_length * embedding_dim
#     keras.layers.Embedding(vocab_size, embedding_dim,
#                            input_length = max_length),
#     keras.layers.LSTM(units = 64, return_sequences = False),
#     keras.layers.Dense(64, activation = 'relu'),
#     keras.layers.Dense(1, activation='sigmoid'),
# ])

# model.summary()
# model.compile(optimizer = 'adam',
#                          loss = 'binary_crossentropy',
#                          metrics = ['accuracy'])

# 2,构建单层单向的LSTM模型(double_rnn_model)
embedding_dim = 16
batch_size = 512

model = keras.models.Sequential([
    # 1. define matrix: [vocab_size, embedding_dim]
    # 2. [1,2,3,4..], max_length * embedding_dim
    # 3. batch_size * max_length * embedding_dim
    keras.layers.Embedding(vocab_size, embedding_dim,
                           input_length = max_length),
    keras.layers.Bidirectional(
        keras.layers.LSTM(
            units = 64, return_sequences = True)),
    keras.layers.Bidirectional(
        keras.layers.LSTM(
            units = 64, return_sequences = False)),
    keras.layers.Dense(64, activation = 'relu'),
    keras.layers.Dense(1, activation='sigmoid'),
])

model.summary()
model.compile(optimizer = 'adam',
              loss = 'binary_crossentropy',
              metrics = ['accuracy'])

# 四,训练模型
history_single_rnn = model.fit(
    train_data, train_labels,
    epochs = 30,
    batch_size = batch_size,
    validation_split = 0.2)

# 五,打印训练图表
def plot_learning_curves(history, label, epochs, min_value, max_value):
    data = {}
    data[label] = history.history[label]
    data['val_'+label] = history.history['val_'+label]
    pd.DataFrame(data).plot(figsize=(8, 5))
    plt.grid(True)
    plt.axis([0, epochs, min_value, max_value])
    plt.show()
    
plot_learning_curves(history_single_rnn, 'accuracy', 30, 0, 1)
plot_learning_curves(history_single_rnn, 'loss', 30, 0, 1)

# 六,打印测试集结果
res_test = model.evaluate(
    test_data, test_labels,
    batch_size = batch_size,
    verbose = 0)

print(res_test)
# tensorflow # keras # nlp # 自然语言处理 # lstm
Logo
技术共进，成长同行——讯飞AI开发者社区

技术共进，成长同行——讯飞AI开发者社区

更多推荐

第75篇：AI+金融：智能风控、量化交易与个性化理财

摘要：本文系统性地深入探讨人工智能在金融领域的革命性应用。我们首先解析智能风控：AI如何利用XGBoost、LightGBM、深度神经网络（DNN）与图神经网络（GNN），融合多维度数据（征信、行为、社交、设备），实现信贷反欺诈、信用评分、反洗钱与异常交易检测，提升风险识别精度与效率。详解量化交易：AI如何通过LSTM、Transformer、强化学习（RL）分析海量市场数据（行情、新闻、财报），

avatar 讯飞AI开发者社区

AIoT（人工智能物联网）：融合范式下的技术演进、系统架构与产业变革

AIoT并非简单的技术叠加，而是一种全新的系统性范式，其核心在于将AI的感知、学习、推理与决策能力内化于物联网的感知、连接与执行体系之中，实现从“万物互联”向“万物智联”的根本性跃迁。通过分析多维度的行业报告、技术白皮书与实践案例，本文揭示了AIoT在智慧城市、智能制造、智慧医疗、智慧农业等领域的深刻变革，并探讨了其在终端AI化、传输多样化、感知融合化等前沿趋势下的产业生态演化。物联网（IoT）的

avatar 讯飞AI开发者社区

企业AI Agent的情感智能：提升员工体验与忠诚度

随着人工智能技术的飞速发展，企业AI Agent在企业运营中的应用越来越广泛。本研究的目的在于深入探讨企业AI Agent的情感智能如何对员工体验和忠诚度产生积极影响。具体范围涵盖了情感智能的理论基础、相关算法原理、实际应用场景以及在企业环境中的实践案例等方面。通过研究，旨在为企业提供切实可行的方案，利用AI Agent的情感智能提升员工的工作满意度和忠诚度，进而促进企业的长期发展。本文共分为十个

avatar 讯飞AI开发者社区