@mhg/blog
Version:
5 lines • 159 kB
HTML
<!DOCTYPE html><html lang="zh-CN"><head hexo-theme="https://github.com/volantis-x/hexo-theme-volantis/#6.0.0-alpha.0"><meta name="generator" content="Hexo 6.3.0"><meta name="Volantis" content="6.0.0-alpha.0"><meta charset="utf-8"><link rel="canonical" href="https://blog.mhuig.top/p/48f700eb/"><meta http-equiv="x-dns-prefetch-control" content="on"><link rel="dns-prefetch" href="https://static.mhuig.top"><link rel="preconnect" href="https://static.mhuig.top" crossorigin><meta name="renderer" content="webkit"><meta name="force-rendering" content="webkit"><meta http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1"><meta name="HandheldFriendly" content="True"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"><meta content="black-translucent" name="apple-mobile-web-app-status-bar-style"><meta content="telephone=no" name="format-detection"><script>function registerServiceWorker(e){"localhost"!=window.location.hostname&&"serviceWorker"in navigator&&navigator.serviceWorker.register(`${e}`).then((function(e){"localhost"==window.location.hostname&&(e.onupdatefound=function(){var r=e.installing;r.onstatechange=function(){switch(r.state){case"installed":navigator.serviceWorker.controller?console.log("Updated Service Worker."):console.log("Service Worker Sucess!");break;case"redundant":console.log("The installing service worker became redundant.")}}})})).catch((function(e){console.error("Error during service worker registration:",e),"undefined"==typeof swinstallretry&&(swinstallretry=1,registerServiceWorker("/jquery.js?v=0258ff4f79"))}))}registerServiceWorker("/jquery.js?v=0258ff4f79")</script><link rel="apple-touch-icon" sizes="180x180" href="/lib/favicon/apple-touch-icon.png?v=e035908f18"><link rel="icon" type="image/png" sizes="32x32" href="/lib/favicon/favicon-32x32.png?v=b9d9ae54cc"><link rel="icon" type="image/png" sizes="192x192" href="/lib/favicon/android-chrome-192x192.png?v=8bf02188e3"><link rel="icon" type="image/png" sizes="144x144" href="/lib/favicon/android-chrome-144x144.png?v=5c8eadcbb7"><link rel="icon" type="image/png" sizes="16x16" href="/lib/favicon/favicon-16x16.png?v=f896f2f634"><link rel="manifest" href="/lib/favicon/site.webmanifest?v=f138de5899"><link rel="mask-icon" href="/lib/favicon/safari-pinned-tab.svg?v=87d61e51a5" color="#5bbad5"><meta name="apple-mobile-web-app-title" content="MHuiG's Blog"><meta name="application-name" content="MHuiG's Blog"><meta name="msapplication-TileColor" content="#87ceeb"><meta name="msapplication-TileImage" content="/lib/favicon/mstile-144x144.png"><meta name="theme-color" content="#87ceeb" media="(prefers-color-scheme: light)"><meta name="theme-color" content="#21232f" media="(prefers-color-scheme: dark)"><link href="/opensearch.xml?v=2f4fd4e7e0" rel="search" title="MHuiG's Blog" type="application/opensearchdescription+xml"><link rel="preload" href="/css/style.css?v=2d24f6fe3e" as="style"><link rel="preload" href="https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/fonts/VarelaRound/VarelaRound-Regular.ttf" as="font" type="font/ttf" crossorigin="anonymous"><link rel="alternate" href="/atom.xml?v=fe8205151a" title="MHuiG" type="application/atom+xml"><link rel="alternate" href="/rss2.xml?v=6b8706d024" title="MHuiG" type="application/rss+xml"><title>BERT 预训练模型及其应用案例 - MHuiG</title><meta name="keywords" content="NLP,Blog, IT, BigData, MHuiG, @MHuiG, iMHuiG, 博客, 梦幻岛, Neverland, 技术博客"><meta desc name="description" content="预训练模型最开始是在图像领域提出的，获得了良好的效果，近几年才被广泛应用到自然语言处理各项任务中。
(1) 2003 年 Bengio 提出神经网络语言模型 NNLM，从此统一了 NLP 的特征形式 ——Embedding；
(2) 2013 年 Mikolov 提出词向量 Word2vec，延续 NNLM ... - MHuiG - MHuiG"><meta property="og:type" content="article"><meta property="og:title" content="BERT 预训练模型及其应用案例"><meta property="og:url" content="https://blog.mhuig.top/p/48f700eb/"><meta property="og:site_name" content="MHuiG"><meta property="og:description" content="预训练模型最开始是在图像领域提出的，获得了良好的效果，近几年才被广泛应用到自然语言处理各项任务中。 (1) 2003 年 Bengio 提出神经网络语言模型 NNLM，从此统一了 NLP 的特征形式 ——Embedding； (2) 2013 年 Mikolov 提出词向量 Word2vec，延续 NNLM 又引入了大规模预训练（Pretrain）的思路； (3) 2017 年 Vaswan"><meta property="og:locale" content="zh_CN"><meta property="og:image" content="https://blog.mhuig.top/lib/favicon/android-chrome-192x192.png"><meta property="article:published_time" content="2020-11-05T07:47:27.000Z"><meta property="article:modified_time" content="2020-11-05T07:47:27.000Z"><meta property="article:author" content="MHuiG"><meta property="article:tag" content="NLP"><meta name="twitter:card" content="summary"><meta name="twitter:image" content="https://blog.mhuig.top/lib/favicon/android-chrome-192x192.png"><meta name="twitter:creator" content="@iMHuiG"><meta name="twitter:site" content="@iMHuiG"><style>#safearea{display:none}.post-story+.post-story{content-visibility:auto;contain-intrinsic-size:10px 500px}:root{--color-site-body:#87ceeb;--color-site-bg:#87ceeb;--color-site-inner:#555;--color-site-footer:#666;--color-card:#fff;--color-text:#444;--color-block:#f6f6f6;--color-inlinecode:#d56d28;--color-codeblock:#fff7ea;--color-h1:#3a3a3a;--color-h2:#3a3a3a;--color-h3:#333;--color-h4:#444;--color-h5:#555;--color-h6:#666;--color-p:#444;--color-list:#666;--color-list-hl:#1a78c2;--color-meta:#888;--color-read-bkg:#e0d8c8;--color-read-post:#f8f1e2;--color-copyright-bkg:#f5f5f5}*{box-sizing:border-box;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;outline:0;margin:0;padding:0}::-webkit-scrollbar{height:4px;width:4px}::-webkit-scrollbar-track-piece{background:0 0}::-webkit-scrollbar-thumb{background:#2196f3;cursor:pointer;border-radius:2px;-webkit-border-radius:2px}::-webkit-scrollbar-thumb:hover{background:#ff5722}html{color:var(--color-text);width:100%;height:100%;font-family:"Varela Round","PingFang SC","Microsoft YaHei",Helvetica,Arial,Menlo,Monaco,monospace,sans-serif;font-size:16px}html>::-webkit-scrollbar{height:4px;width:4px}html>::-webkit-scrollbar-track-piece{background:0 0}html>::-webkit-scrollbar-thumb{background:#2196f3;cursor:pointer;border-radius:2px;-webkit-border-radius:2px}html>::-webkit-scrollbar-thumb:hover{background:#ff5722}body{background-color:var(--color-site-body);text-rendering:optimizelegibility;-webkit-tap-highlight-color:transparent;line-height:1.6;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}body.modal-active{overflow:hidden}@media screen and (max-width:680px){body.modal-active{position:fixed;top:0;right:0;bottom:0;left:0}}a{color:#2196f3;cursor:pointer;text-decoration:none;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}a:hover{color:#ff5722}a:active,a:hover{outline:0}ol,ul{padding-left:0}ol li,ul li{list-style:none}header{display:-webkit-box;display:-moz-box;display:block}img{border:0;background:0 0;max-width:100%}svg:not(:root){overflow:hidden}hr{-moz-box-sizing:content-box;box-sizing:content-box;-webkit-box-sizing:content-box;-moz-box-sizing:content-box;height:0;border:0;border-radius:1px;-webkit-border-radius:1px;border-bottom:1px solid rgba(68,68,68,.1)}button,input{color:inherit;font:inherit;margin:0}button{overflow:visible;text-transform:none;-webkit-appearance:button;cursor:pointer}@supports (backdrop-filter:blur(20px)){.blur{background:rgba(255,255,255,.9)!important;backdrop-filter:saturate(200%) blur(20px)}}.shadow{box-shadow:0 1px 2px 0 rgba(0,0,0,.1);-webkit-box-shadow:0 1px 2px 0 rgba(0,0,0,.1)}.shadow.floatable{transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}.shadow.floatable:hover{box-shadow:0 2px 4px 0 rgba(0,0,0,.1),0 4px 8px 0 rgba(0,0,0,.1),0 8px 16px 0 rgba(0,0,0,.1);-webkit-box-shadow:0 2px 4px 0 rgba(0,0,0,.1),0 4px 8px 0 rgba(0,0,0,.1),0 8px 16px 0 rgba(0,0,0,.1)}#l_cover{min-height:64px}.cover-wrapper{top:0;left:0;max-width:100%;height:100vh;display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;flex-wrap:nowrap;-webkit-flex-wrap:nowrap;-khtml-flex-wrap:nowrap;-moz-flex-wrap:nowrap;-o-flex-wrap:nowrap;-ms-flex-wrap:nowrap;-webkit-box-direction:normal;-moz-box-direction:normal;-webkit-box-orient:vertical;-moz-box-orient:vertical;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;align-items:center;align-self:center;align-content:center;color:var(--color-site-inner);padding:0 16px;user-select:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;position:relative;overflow:hidden;margin-bottom:-100px}.cover-wrapper .cover-bg{position:absolute;width:100%;height:100%;background-position:center;background-size:cover;-webkit-background-size:cover;-moz-background-size:cover}.cover-wrapper .cover-bg.lazyload:not(.loaded){opacity:0;-webkit-opacity:0;-moz-opacity:0}.cover-wrapper .cover-bg.lazyload.loaded{animation-delay:0s;animation-duration:.5s;animation-fill-mode:forwards;animation-timing-function:ease-out;animation-name:fadeIn}@-moz-keyframes fadeIn{0%{opacity:0;-webkit-opacity:0;-moz-opacity:0;filter:blur(12px);transform:scale(1.02);-webkit-transform:scale(1.02);-khtml-transform:scale(1.02);-moz-transform:scale(1.02);-o-transform:scale(1.02);-ms-transform:scale(1.02)}100%{opacity:1;-webkit-opacity:1;-moz-opacity:1}}@-webkit-keyframes fadeIn{0%{opacity:0;-webkit-opacity:0;-moz-opacity:0;filter:blur(12px);transform:scale(1.02);-webkit-transform:scale(1.02);-khtml-transform:scale(1.02);-moz-transform:scale(1.02);-o-transform:scale(1.02);-ms-transform:scale(1.02)}100%{opacity:1;-webkit-opacity:1;-moz-opacity:1}}@-o-keyframes fadeIn{0%{opacity:0;-webkit-opacity:0;-moz-opacity:0;filter:blur(12px);transform:scale(1.02);-webkit-transform:scale(1.02);-khtml-transform:scale(1.02);-moz-transform:scale(1.02);-o-transform:scale(1.02);-ms-transform:scale(1.02)}100%{opacity:1;-webkit-opacity:1;-moz-opacity:1}}@keyframes fadeIn{0%{opacity:0;-webkit-opacity:0;-moz-opacity:0;filter:blur(12px);transform:scale(1.02);-webkit-transform:scale(1.02);-khtml-transform:scale(1.02);-moz-transform:scale(1.02);-o-transform:scale(1.02);-ms-transform:scale(1.02)}100%{opacity:1;-webkit-opacity:1;-moz-opacity:1}}.cover-wrapper .cover-body{z-index:1;position:relative;width:100%;height:100%}.cover-wrapper#full{height:calc(100vh + 100px);padding-bottom:100px}.cover-wrapper#half{max-height:640px;min-height:400px;height:calc(36vh - 64px + 200px)}.cover-wrapper #scroll-down{width:100%;height:64px;position:absolute;bottom:100px;text-align:center;cursor:pointer}.cover-wrapper #scroll-down .scroll-down-effects{color:#fff;font-size:24px;line-height:64px;position:absolute;width:24px;left:calc(50% - 12px);text-shadow:0 1px 2px rgba(0,0,0,.1);animation:scroll-down-effect 1.5s infinite;-webkit-animation:scroll-down-effect 1.5s infinite;-khtml-animation:scroll-down-effect 1.5s infinite;-moz-animation:scroll-down-effect 1.5s infinite;-o-animation:scroll-down-effect 1.5s infinite;-ms-animation:scroll-down-effect 1.5s infinite}@-moz-keyframes scroll-down-effect{0%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}50%{top:-16px;opacity:.4;-webkit-opacity:0.4;-moz-opacity:0.4}100%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}}@-webkit-keyframes scroll-down-effect{0%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}50%{top:-16px;opacity:.4;-webkit-opacity:0.4;-moz-opacity:0.4}100%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}}@-o-keyframes scroll-down-effect{0%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}50%{top:-16px;opacity:.4;-webkit-opacity:0.4;-moz-opacity:0.4}100%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}}@keyframes scroll-down-effect{0%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}50%{top:-16px;opacity:.4;-webkit-opacity:0.4;-moz-opacity:0.4}100%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}}.cover-wrapper .cover-body{margin-top:64px;margin-bottom:100px}.cover-wrapper .cover-body,.cover-wrapper .cover-body .bottom,.cover-wrapper .cover-body .top{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;-webkit-box-direction:normal;-moz-box-direction:normal;-webkit-box-orient:vertical;-moz-box-orient:vertical;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;align-items:center;justify-content:center;-webkit-justify-content:center;-khtml-justify-content:center;-moz-justify-content:center;-o-justify-content:center;-ms-justify-content:center;max-width:100%}.cover-wrapper .cover-body .bottom{margin-top:32px}.cover-wrapper .cover-body .title{font-family:"Varela Round","PingFang SC","Microsoft YaHei",Helvetica,Arial,Helvetica,monospace;font-size:3.125rem;line-height:1.2;text-shadow:0 1px 2px rgba(0,0,0,.1)}.cover-wrapper .cover-body .subtitle{font-size:20px}.cover-wrapper .cover-body .logo{max-height:120px;max-width:calc(100% - 4 * 16px)}@media screen and (min-height:1024px){.cover-wrapper .cover-body .title{font-size:3rem}.cover-wrapper .cover-body .subtitle{font-size:1.05rem}.cover-wrapper .cover-body .logo{max-height:150px}}.cover-wrapper .cover-body .m_search{position:relative;max-width:calc(100% - 16px);width:320px;vertical-align:middle}.cover-wrapper .cover-body .m_search .form{position:relative;display:-webkit-box;display:-moz-box;display:block;width:100%}.cover-wrapper .cover-body .m_search .icon,.cover-wrapper .cover-body .m_search .input{transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}.cover-wrapper .cover-body .m_search .icon{position:absolute;display:-webkit-box;display:-moz-box;display:block;line-height:2.5rem;width:32px;top:0;left:5px;color:rgba(68,68,68,.75)}.cover-wrapper .cover-body .m_search .input{display:-webkit-box;display:-moz-box;display:block;height:2.5rem;width:100%;box-shadow:none;-webkit-box-shadow:none;box-sizing:border-box;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;font-size:.875rem;-webkit-appearance:none;padding-left:36px;border-radius:1.4rem;-webkit-border-radius:1.4rem;background:rgba(255,255,255,.6);backdrop-filter:blur(10px);border:none;color:var(--color-text)}@media screen and (max-width:500px){.cover-wrapper .cover-body .m_search .input{padding-left:36px}}.cover-wrapper .cover-body .m_search .input:hover{background:rgba(255,255,255,.8)}.cover-wrapper .cover-body .m_search .input:focus{background:#fff}.cover-wrapper .list-h{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;-webkit-box-direction:normal;-moz-box-direction:normal;-webkit-box-orient:horizontal;-moz-box-orient:horizontal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;flex-wrap:wrap;-webkit-flex-wrap:wrap;-khtml-flex-wrap:wrap;-moz-flex-wrap:wrap;-o-flex-wrap:wrap;-ms-flex-wrap:wrap;align-items:stretch;border-radius:4px;-webkit-border-radius:4px;user-select:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none}.cover-wrapper .list-h a{-webkit-box-flex:1;-moz-box-flex:1;-webkit-flex:1 0;-ms-flex:1 0;flex:1 0;display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;font-weight:600}.cover-wrapper .list-h a img{display:-webkit-box;display:-moz-box;display:block;border-radius:2px;-webkit-border-radius:2px;margin:4px;min-width:40px;max-width:44px}@media screen and (max-width:768px){.cover-wrapper .list-h a img{min-width:36px;max-width:40px}}@media screen and (max-width:500px){.cover-wrapper .list-h a img{margin:2px 4px;min-width:32px;max-width:36px}}@media screen and (max-width:375px){.cover-wrapper .list-h a img{min-width:28px;max-width:32px}}.cover-wrapper{max-width:100%}.cover-wrapper.search .bottom .menu{margin-top:16px}.cover-wrapper.search .bottom .menu .list-h a{white-space:nowrap;-webkit-box-direction:normal;-moz-box-direction:normal;-webkit-box-orient:horizontal;-moz-box-orient:horizontal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;align-items:baseline;padding:2px;margin:4px;color:var(--color-site-inner);opacity:.75;-webkit-opacity:0.75;-moz-opacity:0.75;text-shadow:0 1px 2px rgba(0,0,0,.05);border-bottom:2px solid transparent}.cover-wrapper.search .bottom .menu .list-h a i{margin-right:4px}.cover-wrapper.search .bottom .menu .list-h a p{font-size:.9375rem}.cover-wrapper.search .bottom .menu .list-h a.active,.cover-wrapper.search .bottom .menu .list-h a:active,.cover-wrapper.search .bottom .menu .list-h a:hover{opacity:1;-webkit-opacity:1;-moz-opacity:1;border-bottom:2px solid var(--color-site-inner)}.cover-wrapper.dock .menu,.cover-wrapper.featured .menu,.cover-wrapper.focus .menu{border-radius:6px;-webkit-border-radius:6px}.cover-wrapper.dock .menu .list-h a,.cover-wrapper.featured .menu .list-h a,.cover-wrapper.focus .menu .list-h a{-webkit-box-direction:normal;-moz-box-direction:normal;-webkit-box-orient:vertical;-moz-box-orient:vertical;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;align-items:center;padding:12px;line-height:24px;border-radius:4px;-webkit-border-radius:4px;border-bottom:none;text-align:center;align-content:flex-end;color:rgba(68,68,68,.7);font-size:1.5rem}@media screen and (max-width:500px){.cover-wrapper.dock .menu .list-h a,.cover-wrapper.featured .menu .list-h a,.cover-wrapper.focus .menu .list-h a{padding:12px 8px}}.cover-wrapper.dock .menu .list-h a i,.cover-wrapper.featured .menu .list-h a i,.cover-wrapper.focus .menu .list-h a i{margin:8px}.cover-wrapper.dock .menu .list-h a p,.cover-wrapper.featured .menu .list-h a p,.cover-wrapper.focus .menu .list-h a p{font-size:.875rem}.cover-wrapper.dock .menu .list-h a.active,.cover-wrapper.featured .menu .list-h a.active,.cover-wrapper.focus .menu .list-h a.active{background:var(--color-card);backdrop-filter:none}.cover-wrapper.dock .menu .list-h a.active i,.cover-wrapper.dock .menu .list-h a.active i+p,.cover-wrapper.featured .menu .list-h a.active i,.cover-wrapper.featured .menu .list-h a.active i+p,.cover-wrapper.focus .menu .list-h a.active i,.cover-wrapper.focus .menu .list-h a.active i+p{color:#2196f3}.cover-wrapper.dock .menu .list-h a.active img+p,.cover-wrapper.featured .menu .list-h a.active img+p,.cover-wrapper.focus .menu .list-h a.active img+p{color:var(--color-text)}.cover-wrapper.dock .menu .list-h a:hover,.cover-wrapper.featured .menu .list-h a:hover,.cover-wrapper.focus .menu .list-h a:hover{background:var(--color-card)}.cover-wrapper.featured .menu .list-h{margin:-2px}.cover-wrapper.featured .menu .list-h a{margin:2px;background:rgba(255,255,255,.5)}@supports (backdrop-filter:blur(20px)){.cover-wrapper.featured .menu .list-h a{background:rgba(255,255,255,.5);backdrop-filter:saturate(200%) blur(20px)}}@media (prefers-color-scheme:dark){:root{--color-mode:'dark'}:root:not([color-scheme]){--color-site-body:#121212;--color-read-bkg:#21232f;--color-read-post:#252d38;--color-site-bg:#21232f;--color-site-inner:#efefef;--color-site-footer:#666;--color-card:#252d38;--color-text:rgba(238,238,238,0.871);--color-block:rgba(68,68,68,0.65);--color-codeblock:rgba(68,68,68,0.65);--color-inlinecode:#d56d28;--color-h1:rgba(255,255,255,0.871);--color-h2:rgba(255,255,255,0.871);--color-h3:rgba(255,255,255,0.6);--color-h4:rgba(255,255,255,0.6);--color-h5:rgba(255,255,255,0.6);--color-h6:rgba(255,255,255,0.6);--color-p:rgba(217,217,217,0.871);--color-list:rgba(217,217,217,0.871);--color-list-hl:#4dabf5;--color-meta:rgba(191,191,191,0.871);--color-link:rgba(191,191,191,0.871);--color-copyright-bkg:#21252b}:root:not([color-scheme]) img{filter:brightness(70%)!important}:root:not([color-scheme]) .blur{background:rgba(33,35,47,.9)!important}:root:not([color-scheme]) .white-box.blur{background:rgba(37,45,56,.9)!important}:root:not([color-scheme]) .nav-main .u-search-input{background:var(--color-card)!important}:root:not([color-scheme]) #l_main .article .prev-next>a{background:var(--color-block)!important}:root:not([color-scheme]) #l_main .article .prev-next>a:hover{background:var(--color-site-bg)!important}:root:not([color-scheme]) .article blockquote{background:var(--color-block)!important}:root:not([color-scheme]) .article-title a{color:var(--color-h1)!important}:root:not([color-scheme]) details>summary{color:var(--color-p)!important;background:var(--color-site-bg)!important}:root:not([color-scheme]) details{border:1px solid var(--color-site-bg)!important;background:var(--color-site-bg)!important}:root:not([color-scheme]) #u-search .modal,:root:not([color-scheme]) #u-search .modal-body,:root:not([color-scheme]) #u-search .modal-header{background:var(--color-card)!important}:root:not([color-scheme]) #u-search .modal-body .modal-results .result:hover{background:var(--color-block)!important}:root:not([color-scheme]) .u-search-input:hover{background:var(--color-block)!important}:root:not([color-scheme]) .u-search-input:focus{background:var(--color-site-body)!important}}[color-scheme=dark]{--color-site-body:#121212;--color-read-bkg:#21232f;--color-read-post:#252d38;--color-site-bg:#21232f;--color-site-inner:#efefef;--color-site-footer:#666;--color-card:#252d38;--color-text:rgba(238,238,238,0.871);--color-block:rgba(68,68,68,0.65);--color-codeblock:rgba(68,68,68,0.65);--color-inlinecode:#d56d28;--color-h1:rgba(255,255,255,0.871);--color-h2:rgba(255,255,255,0.871);--color-h3:rgba(255,255,255,0.6);--color-h4:rgba(255,255,255,0.6);--color-h5:rgba(255,255,255,0.6);--color-h6:rgba(255,255,255,0.6);--color-p:rgba(217,217,217,0.871);--color-list:rgba(217,217,217,0.871);--color-list-hl:#4dabf5;--color-meta:rgba(191,191,191,0.871);--color-link:rgba(191,191,191,0.871);--color-copyright-bkg:#21252b}[color-scheme=dark] img{filter:brightness(70%)!important}[color-scheme=dark] .blur{background:rgba(33,35,47,.9)!important}[color-scheme=dark] .white-box.blur{background:rgba(37,45,56,.9)!important}[color-scheme=dark] .nav-main .u-search-input{background:var(--color-card)!important}[color-scheme=dark] #l_main .article .prev-next>a{background:var(--color-block)!important}[color-scheme=dark] #l_main .article .prev-next>a:hover{background:var(--color-site-bg)!important}[color-scheme=dark] .article blockquote{background:var(--color-block)!important}[color-scheme=dark] .article-title a{color:var(--color-h1)!important}[color-scheme=dark] details>summary{color:var(--color-p)!important;background:var(--color-site-bg)!important}[color-scheme=dark] details{border:1px solid var(--color-site-bg)!important;background:var(--color-site-bg)!important}[color-scheme=dark] #u-search .modal,[color-scheme=dark] #u-search .modal-body,[color-scheme=dark] #u-search .modal-header{background:var(--color-card)!important}[color-scheme=dark] #u-search .modal-body .modal-results .result:hover{background:var(--color-block)!important}[color-scheme=dark] .u-search-input:hover{background:var(--color-block)!important}[color-scheme=dark] .u-search-input:focus{background:var(--color-site-body)!important}@media screen and (max-width:500px){[color-scheme=dark] .l_header .m_search{background:var(--color-site-bg)!important}}@font-face{font-family:'Varela Round';src:url("");font-weight:'normal';font-style:'normal';font-display:swap}@font-face{font-family:'Varela Round';src:url("https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/fonts/VarelaRound/VarelaRound-Regular.ttf");font-weight:'normal';font-style:'normal';font-display:swap}.l_header{position:fixed;z-index:1000;top:0;width:100%;height:64px;background:var(--color-card);box-shadow:0 1px 2px 0 rgba(0,0,0,.1);-webkit-box-shadow:0 1px 2px 0 rgba(0,0,0,.1)}.l_header.auto{transition:opacity .4s ease;-webkit-transition:opacity .4s ease;-khtml-transition:opacity 0.4s ease;-moz-transition:opacity .4s ease;-o-transition:opacity .4s ease;-ms-transition:opacity .4s ease;visibility:hidden}.l_header.auto.show{opacity:1!important;-webkit-opacity:1!important;-moz-opacity:1!important;visibility:visible}.l_header .container{margin-left:16px;margin-right:16px}.l_header #wrapper{height:100%;user-select:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none}.l_header #wrapper .nav-main,.l_header #wrapper .nav-sub{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;flex-wrap:nowrap;-webkit-flex-wrap:nowrap;-khtml-flex-wrap:nowrap;-moz-flex-wrap:nowrap;-o-flex-wrap:nowrap;-ms-flex-wrap:nowrap;justify-content:space-between;-webkit-justify-content:space-between;-khtml-justify-content:space-between;-moz-justify-content:space-between;-o-justify-content:space-between;-ms-justify-content:space-between;align-items:center}.l_header #wrapper .nav-main{transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}.l_header #wrapper.sub .nav-main{transform:translateY(-64px);-webkit-transform:translateY(-64px);-khtml-transform:translateY(-64px);-moz-transform:translateY(-64px);-o-transform:translateY(-64px);-ms-transform:translateY(-64px)}.l_header #wrapper .nav-sub{transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease;opacity:0;-webkit-opacity:0;-moz-opacity:0;height:64px;width:calc(100% - 2 * 16px);position:absolute}.l_header #wrapper .nav-sub ::-webkit-scrollbar{display:-webkit-box;display:-moz-box;display:none}@media screen and (min-width:2048px){.l_header #wrapper .nav-sub{max-width:55vw;margin:auto}}.l_header #wrapper.sub .nav-sub{opacity:1;-webkit-opacity:1;-moz-opacity:1}.l_header #wrapper .title{position:relative;color:var(--color-text);padding-left:24px;max-height:64px}.l_header #wrapper .nav-main .title{white-space:nowrap;overflow:hidden;text-overflow:ellipsis;flex-shrink:0;line-height:64px;padding:0 24px;font-size:1.25rem;font-family:"Varela Round","PingFang SC","Microsoft YaHei",Helvetica,Arial,Helvetica,monospace}.l_header #wrapper .nav-main .title img{height:64px}.l_header .nav-sub{max-width:1080px;margin:auto}.l_header .nav-sub .title{font-weight:700;font-family:"Varela Round","PingFang SC","Microsoft YaHei",Helvetica,Arial,Menlo,Monaco,monospace,sans-serif;line-height:1.2;max-height:64px;white-space:normal;flex-shrink:1}.l_header .switcher{display:-webkit-box;display:-moz-box;display:none;line-height:64px;align-items:center}.l_header .switcher .s-toc{display:-webkit-box;display:-moz-box;display:none}@media screen and (max-width:768px){.l_header .switcher .s-toc{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex}}.l_header .switcher>li{height:48px;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease;margin:2px}@media screen and (max-width:500px){.l_header .switcher>li{margin:0 1px;height:48px}}.l_header .switcher>li>a{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;justify-content:center;-webkit-justify-content:center;-khtml-justify-content:center;-moz-justify-content:center;-o-justify-content:center;-ms-justify-content:center;align-items:center;width:48px;height:48px;padding:.85em 1.1em;border-radius:100px;-webkit-border-radius:100px;border:none;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease;color:#2196f3}.l_header .switcher>li>a:hover{border:none}.l_header .switcher>li>a.active,.l_header .switcher>li>a:active{border:none;background:var(--color-site-bg)}@media screen and (max-width:500px){.l_header .switcher>li>a{width:36px;height:48px}}.l_header .nav-sub .switcher{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex}.l_header .m_search{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;height:64px;width:240px;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}@media screen and (max-width:1024px){.l_header .m_search{width:44px;min-width:44px}.l_header .m_search input::placeholder{opacity:0;-webkit-opacity:0;-moz-opacity:0}.l_header .m_search:hover{width:240px}.l_header .m_search:hover input::placeholder{opacity:1;-webkit-opacity:1;-moz-opacity:1}}@media screen and (min-width:500px){.l_header .m_search:hover .input{width:100%}.l_header .m_search:hover .input::placeholder{opacity:1;-webkit-opacity:1;-moz-opacity:1}}@media screen and (max-width:500px){.l_header .m_search{min-width:0}.l_header .m_search input::placeholder{opacity:1;-webkit-opacity:1;-moz-opacity:1}}.l_header .m_search .form{position:relative;display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;width:100%;align-items:center}.l_header .m_search .icon{position:absolute;width:36px;left:5px;color:var(--color-meta)}@media screen and (max-width:500px){.l_header .m_search .icon{display:-webkit-box;display:-moz-box;display:none}}.l_header .m_search .input{display:-webkit-box;display:-moz-box;display:block;padding-top:8px;padding-bottom:8px;line-height:1.3;width:100%;color:var(--color-text);background:#fafafa;box-shadow:none;-webkit-box-shadow:none;box-sizing:border-box;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;padding-left:40px;font-size:.875rem;border-radius:8px;-webkit-border-radius:8px;border:none;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}@media screen and (min-width:500px){.l_header .m_search .input:focus{box-shadow:0 4px 8px 0 rgba(0,0,0,.1);-webkit-box-shadow:0 4px 8px 0 rgba(0,0,0,.1)}}@media screen and (max-width:500px){.l_header .m_search .input{background:var(--color-block);padding-left:8px;border:none}.l_header .m_search .input:focus,.l_header .m_search .input:hover{border:none}}@media (max-width:500px){.l_header .m_search{left:0;width:0;overflow:hidden;position:absolute;background:#fff;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}.l_header .m_search .input{border-radius:32px;-webkit-border-radius:32px;margin-left:16px;padding-left:16px}.l_header.z_search-open .m_search{width:100%}.l_header.z_search-open .m_search .input{width:calc(100% - 120px)}}ul.m-pc>li>a{color:inherit;border-bottom:2px solid transparent}ul.m-pc>li>a.active,ul.m-pc>li>a:active{border-bottom:2px solid #2196f3}ul.list-v li:hover>ul.list-v,ul.m-pc li:hover>ul.list-v{display:-webkit-box;display:-moz-box;display:block}ul.nav-list-h{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;align-items:stretch}ul.nav-list-h>li{position:relative;justify-content:center;-webkit-justify-content:center;-khtml-justify-content:center;-moz-justify-content:center;-o-justify-content:center;-ms-justify-content:center;height:100%;line-height:2.4;border-radius:4px;-webkit-border-radius:4px}ul.nav-list-h>li>a{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;font-weight:600}ul.list-v{z-index:1;display:-webkit-box;display:-moz-box;display:none;position:absolute;background:var(--color-card);box-shadow:0 2px 4px 0 rgba(0,0,0,.08),0 4px 8px 0 rgba(0,0,0,.08),0 8px 16px 0 rgba(0,0,0,.08);-webkit-box-shadow:0 2px 4px 0 rgba(0,0,0,.08),0 4px 8px 0 rgba(0,0,0,.08),0 8px 16px 0 rgba(0,0,0,.08);margin-top:-6px;border-radius:4px;-webkit-border-radius:4px;padding:8px 0}ul.list-v.show{display:-webkit-box;display:-moz-box;display:block}ul.list-v hr{margin-top:8px;margin-bottom:8px}ul.list-v>li{white-space:nowrap;word-break:keep-all}ul.list-v>li.header{font-size:.78125rem;font-weight:700;line-height:2em;color:var(--color-meta);margin:8px 16px 4px}ul.list-v>li.header i{margin-right:8px}ul.list-v>li ul{margin-left:0;display:-webkit-box;display:-moz-box;display:none;margin-top:-40px}ul.list-v .aplayer-container{min-height:64px;padding:6px 16px}ul.list-v>li>a{transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease;display:-webkit-box;display:-moz-box;display:block;color:var(--color-list);font-size:.875rem;font-weight:700;line-height:36px;padding:0 20px 0 16px;text-overflow:ellipsis;margin:0 4px;border-radius:4px;-webkit-border-radius:4px}@media screen and (max-width:1024px){ul.list-v>li>a{line-height:40px}}ul.list-v>li>a>i{margin-right:8px}ul.list-v>li>a.active,ul.list-v>li>a:active{color:var(--color-list-hl)}ul.list-v>li>a:hover{color:var(--color-list-hl);background:var(--color-site-bg)}.l_header .menu>ul>li>a{display:-webkit-box;display:-moz-box;display:block;padding:0 8px}.l_header .menu>ul>li>a>i{margin-right:4px}.l_header ul.nav-list-h>li{color:var(--color-list);line-height:64px}.l_header ul.nav-list-h>li>a{max-height:64px;overflow:hidden;color:inherit}.l_header ul.nav-list-h>li>a.active,.l_header ul.nav-list-h>li>a:active{color:#2196f3}.l_header ul.nav-list-h>li:hover>a{color:var(--color-list-hl)}.l_header ul.nav-list-h>li i.music{animation:rotate-effect 1.5s linear infinite;-webkit-animation:rotate-effect 1.5s linear infinite;-khtml-animation:rotate-effect 1.5s linear infinite;-moz-animation:rotate-effect 1.5s linear infinite;-o-animation:rotate-effect 1.5s linear infinite;-ms-animation:rotate-effect 1.5s linear infinite}@-moz-keyframes rotate-effect{0%{transform:rotate(0);-webkit-transform:rotate(0);-khtml-transform:rotate(0);-moz-transform:rotate(0);-o-transform:rotate(0);-ms-transform:rotate(0)}25%{transform:rotate(90deg);-webkit-transform:rotate(90deg);-khtml-transform:rotate(90deg);-moz-transform:rotate(90deg);-o-transform:rotate(90deg);-ms-transform:rotate(90deg)}50%{transform:rotate(180deg);-webkit-transform:rotate(180deg);-khtml-transform:rotate(180deg);-moz-transform:rotate(180deg);-o-transform:rotate(180deg);-ms-transform:rotate(180deg)}75%{transform:rotate(270deg);-webkit-transform:rotate(270deg);-khtml-transform:rotate(270deg);-moz-transform:rotate(270deg);-o-transform:rotate(270deg);-ms-transform:rotate(270deg)}100%{transform:rotate(360deg);-webkit-transform:rotate(360deg);-khtml-transform:rotate(360deg);-moz-transform:rotate(360deg);-o-transform:rotate(360deg);-ms-transform:rotate(360deg)}}@-webkit-keyframes rotate-effect{0%{transform:rotate(0);-webkit-transform:rotate(0);-khtml-transform:rotate(0);-moz-transform:rotate(0);-o-transform:rotate(0);-ms-transform:rotate(0)}25%{transform:rotate(90deg);-webkit-transform:rotate(90deg);-khtml-transform:rotate(90deg);-moz-transform:rotate(90deg);-o-transform:rotate(90deg);-ms-transform:rotate(90deg)}50%{transform:rotate(180deg);-webkit-transform:rotate(180deg);-khtml-transform:rotate(180deg);-moz-transform:rotate(180deg);-o-transform:rotate(180deg);-ms-transform:rotate(180deg)}75%{transform:rotate(270deg);-webkit-transform:rotate(270deg);-khtml-transform:rotate(270deg);-moz-transform:rotate(270deg);-o-transform:rotate(270deg);-ms-transform:rotate(270deg)}100%{transform:rotate(360deg);-webkit-transform:rotate(360deg);-khtml-transform:rotate(360deg);-moz-transform:rotate(360deg);-o-transform:rotate(360deg);-ms-transform:rotate(360deg)}}@-o-keyframes rotate-effect{0%{transform:rotate(0);-webkit-transform:rotate(0);-khtml-transform:rotate(0);-moz-transform:rotate(0);-o-transform:rotate(0);-ms-transform:rotate(0)}25%{transform:rotate(90deg);-webkit-transform:rotate(90deg);-khtml-transform:rotate(90deg);-moz-transform:rotate(90deg);-o-transform:rotate(90deg);-ms-transform:rotate(90deg)}50%{transform:rotate(180deg);-webkit-transform:rotate(180deg);-khtml-transform:rotate(180deg);-moz-transform:rotate(180deg);-o-transform:rotate(180deg);-ms-transform:rotate(180deg)}75%{transform:rotate(270deg);-webkit-transform:rotate(270deg);-khtml-transform:rotate(270deg);-moz-transform:rotate(270deg);-o-transform:rotate(270deg);-ms-transform:rotate(270deg)}100%{transform:rotate(360deg);-webkit-transform:rotate(360deg);-khtml-transform:rotate(360deg);-moz-transform:rotate(360deg);-o-transform:rotate(360deg);-ms-transform:rotate(360deg)}}@keyframes rotate-effect{0%{transform:rotate(0);-webkit-transform:rotate(0);-khtml-transform:rotate(0);-moz-transform:rotate(0);-o-transform:rotate(0);-ms-transform:rotate(0)}25%{transform:rotate(90deg);-webkit-transform:rotate(90deg);-khtml-transform:rotate(90deg);-moz-transform:rotate(90deg);-o-transform:rotate(90deg);-ms-transform:rotate(90deg)}50%{transform:rotate(180deg);-webkit-transform:rotate(180deg);-khtml-transform:rotate(180deg);-moz-transform:rotate(180deg);-o-transform:rotate(180deg);-ms-transform:rotate(180deg)}75%{transform:rotate(270deg);-webkit-transform:rotate(270deg);-khtml-transform:rotate(270deg);-moz-transform:rotate(270deg);-o-transform:rotate(270deg);-ms-transform:rotate(270deg)}100%{transform:rotate(360deg);-webkit-transform:rotate(360deg);-khtml-transform:rotate(360deg);-moz-transform:rotate(360deg);-o-transform:rotate(360deg);-ms-transform:rotate(360deg)}}.menu-phone li ul.list-v{right:calc(100% - .5 * 16px)}.menu-phone li ul.list-v ul{right:calc(100% - .5 * 16px)}#wrapper{max-width:1080px;margin:auto}@media screen and (min-width:2048px){#wrapper{max-width:55vw}}#wrapper .menu{-webkit-box-flex:1;-moz-box-flex:1;-webkit-flex:1 1;-ms-flex:1 1;flex:1 1;margin:0 16px 0 0}#wrapper .menu .list-v ul{left:calc(100% - .5 * 16px)}.menu-phone{display:-webkit-box;display:-moz-box;display:none;margin-top:16px;right:8px;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}.menu-phone ul{right:calc(100% - .5 * 16px)}@media screen and (max-width:500px){.menu-phone{display:-webkit-box;display:-moz-box;display:block}}.l_header{max-width:65vw;left:calc((100% - 65vw) * .5);border-bottom-left-radius:8px;border-bottom-right-radius:8px}@media screen and (max-width:2048px){.l_header{max-width:1112px;left:calc((100% - 1112px) * .5)}}@media screen and (max-width:1112px){.l_header{left:0;border-radius:0;-webkit-border-radius:0;max-width:100%}}@media screen and (max-width:500px){.l_header .container{margin-left:0;margin-right:0}.l_header #wrapper .nav-main .title{padding-left:16px;padding-right:16px}.l_header #wrapper .nav-sub{width:100%}.l_header #wrapper .nav-sub .title{overflow-y:scroll;margin-top:2px;padding:8px 16px}.l_header #wrapper .switcher{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;margin-right:8px}.l_header .menu{display:-webkit-box;display:-moz-box;display:none}}@media screen and (max-width:500px){.list-v li{max-width:270px}}#u-search{display:-webkit-box;display:-moz-box;display:none;position:fixed;top:0;left:0;width:100%;height:100%;padding:60px 20px;z-index:1001}@media screen and (max-width:680px){#u-search{padding:0}}@media screen and (prefers-color-scheme:dark) and (max-width:500px){.l_header .m_search{background:var(--color-site-bg)!important}}.cover-wrapper .cover-body .subtitle{font-size:.875rem}.fa-brands.color-github,.fa-duotone.color-github,.fa-light.color-github,.fa-regular.color-github,.fa-solid.color-github,.fa-thin.color-github,.fa.color-github,.fad.color-github,.fal.color-github,.far.color-github,.fas.color-github,.iziToast>.iziToast-body .iziToast-icon.color-github{color:#000}.fa-brands.color-friends,.fa-duotone.color-friends,.fa-light.color-friends,.fa-regular.color-friends,.fa-solid.color-friends,.fa-thin.color-friends,.fa.color-friends,.fad.color-friends,.fal.color-friends,.far.color-friends,.fas.color-friends,.iziToast>.iziToast-body .iziToast-icon.color-friends{color:#d31ee9}.fa-brands.color-about,.fa-duotone.color-about,.fa-light.color-about,.fa-regular.color-about,.fa-solid.color-about,.fa-thin.color-about,.fa.color-about,.fad.color-about,.fal.color-about,.far.color-about,.fas.color-about,.iziToast>.iziToast-body .iziToast-icon.color-about{color:#0095ff}.fa-brands.color-travellings,.fa-duotone.color-travellings,.fa-light.color-travellings,.fa-regular.color-travellings,.fa-solid.color-travellings,.fa-thin.color-travellings,.fa.color-travellings,.fad.color-travellings,.fal.color-travellings,.far.color-travellings,.fas.color-travellings,.iziToast>.iziToast-body .iziToast-icon.color-travellings{color:#734ae6}</style><link rel="stylesheet" href="/css/style.css?v=2d24f6fe3e" media="print" onload="this.media="all",this.onload=null"><noscript><link rel="stylesheet" href="/css/style.css?v=2d24f6fe3e"></noscript><script>let userColorScheme=localStorage.getItem("color-scheme");userColorScheme&&document.documentElement.setAttribute("color-scheme",userColorScheme)</script><script>window.MSInputMethodContext&&document.documentMode&&document.write('<style>html{overflow-x: hidden !important;overflow-y: hidden !important;}.kill-ie{text-align:center;height: 100%;margin-top: 15%;margin-bottom: 5500%;}.kill-t{font-size: 2rem;}.kill-c{font-size: 1.2rem;}#l_header,#l_body{display: none;}</style><div class="kill-ie"><span class="kill-t"><b>抱歉，您的浏览器无法访问本站</b></span><br/><span class="kill-c">微软已经于2016年终止了对 Internet Explorer (IE) 10 及更早版本的支持，<br/>继续使用存在极大的安全隐患，请使用当代主流的浏览器进行访问。</span><br/><a target="_blank" rel="noopener" href="https://blogs.windows.com/windowsexperience/2021/05/19/the-future-of-internet-explorer-on-windows-10-is-in-microsoft-edge/"><strong>了解详情 ></strong></a></div>')</script><noscript><style>html{overflow-x:hidden!important;overflow-y:hidden!important}.kill-noscript{text-align:center;height:100%;margin-top:15%;margin-bottom:5500%}.kill-t{font-size:2rem}.kill-c{font-size:1.2rem}#l_body,#l_header{display:none}</style><div class="kill-noscript"> <span class="kill-t"><b>抱歉，您的浏览器无法访问本站</b></span><br> <span class="kill-c">本页面需要浏览器支持（启用）JavaScript</span><br> <a target="_blank" rel="external nofollow noopener noreferrer" href="https://www.baidu.com/s?wd=启用JavaScript"><strong>了解详情 ></strong></a></div></noscript><script>function volantisEventListener(e,t,o){this.type=e,this.f=t,this.ele=o}function volantisDom(e){return e||(e=document.createElement("div")),this.ele=e,this.ele.find=e=>{let t=this.ele.querySelector(e);if(t)return new volantisDom(t)},this.ele.hasClass=e=>this.ele.className.match(new RegExp("(\\s|^)"+e+"(\\s|$)")),this.ele.addClass=e=>(this.ele.classList.add(e),this.ele),this.ele.removeClass=e=>(this.ele.classList.remove(e),this.ele),this.ele.toggleClass=e=>(this.ele.hasClass(e)?this.ele.removeClass(e):this.ele.addClass(e),this.ele),this.ele.on=(e,t,o=1)=>(this.ele.addEventListener(e,t,!1),o&&volantis.EventListener.list.push(new volantisEventListener(e,t,this.ele)),this.ele),this.ele.click=(e,t)=>(this.ele.on("click",e,t),this.ele),this.ele.scroll=(e,t)=>(this.ele.on("scroll",e,t),this.ele),this.ele.html=e=>(this.ele.innerHTML=e,this.ele),this.ele.hide=e=>(this.ele.style.display="none",this.ele),this.ele.show=e=>(this.ele.style.display="block",this.ele),this.ele}function RunItem(){function e(e,t){this.name=t||e.name,this.run=()=>{try{e()}catch(e){console.log(e)}}}this.list=[],this.start=()=>{for(var e=0;e<this.list.length;e++)this.list[e].run()},this.push=(t,o,n=!0)=>{let l=t;n&&(l=()=>{volantis.requestAnimationFrame(t)});var s=new e(l,o);this.list.push(s)},this.remove=e=>{for(let t=0;t<this.list.length;t++){this.list[t].name==e&&this.list.splice(t,1)}}}function errorImgAvatar(e){e.src="https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/placeholder/avatar/round/3442075.svg",e.onerror=null}function errorImgCover(e){e.src="https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/placeholder/cover/76b86c0226ffd.svg",e.onerror=null}window.volantis={},volantis.debug="false",volantis.dom={},volantis.EventListener={},volantis.EventListener.list=[],volantis.EventListener.remove=()=>{volantis.EventListener.list.forEach((function(e){e.ele.removeEventListener(e.type,e.f,!1)})),volantis.EventListener.list=[]},volantis.dom.$=e=>e?new volantisDom(e):null,volantis.pjax={},volantis.pjax.method={complete:new RunItem,error:new RunItem,send:new RunItem},volantis.pjax=Object.assign(volantis.pjax,{push:volantis.pjax.method.complete.push,error:volantis.pjax.method.error.push,send:volantis.pjax.method.send.push}),volantis.rightmenu={},volantis.rightmenu.method={handle:new RunItem},volantis.rightmenu=Object.assign(volantis.rightmenu,{handle:volantis.rightmenu.method.handle.push}),volantis.dark={},volantis.dark.method={toggle:new RunItem},volantis.dark=Object.assign(volantis.dark,{push:volantis.dark.method.toggle.push}),volantis.js=(e,t)=>new Promise((o=>{setTimeout((function(){var n=document.getElementsByTagName("head")[0]||document.documentElement,l=document.createElement("script");if(l.setAttribute("type","text/javascript"),t)if(JSON.stringify(t))for(let e in t)"onload"==e?l[e]=()=>{t[e](),o()}:(l[e]=t[e],l.onload=o);else l.onload=()=>{t(),o()};else l.onload=o;l.setAttribute("src",e),n.appendChild(l)}))})),volantis.css=e=>new Promise((t=>{setTimeout((function(){var o=document.createElement("link");o.rel="stylesheet",o.href=e,o.onload=t,document.getElementsByTagName("head")[0].appendChild(o)}))})),volantis.import={jQuery:()=>"undefined"==typeof jQuery?volantis.js("https://cdn.bootcdn.net/ajax/libs/jquery/3.6.0/jquery.min.js"):new Promise((e=>{e()}))},volantis.requestAnimationFrame=e=>{window.requestAnimationFrame||(window.requestAnimationFrame=window.requestAnimationFrame||window.mozRequestAnimationFrame||window.webkitRequestAnimationFrame),window.requestAnimationFrame(e)},volantis.layoutHelper=(e,t,o)=>{function n(e,t,o){volantis.tempDiv=document.createElement("div"),volantis.tempDiv.innerHTML=t;let n=document.querySelector("#layoutHelper-"+e);n&&(o&&(n.innerHTML=""),n.append(volantis.tempDiv))}o=Object.assign({clean:!1,pjax:!0},o),n(e,t,o.clean),o.pjax&&volantis.pjax.push((()=>{n(e,t,o.clean)}),"layoutHelper-"+e)},volantis.scroll={engine:new RunItem,unengine:new RunItem},volantis.scroll=Object.assign(volantis.scroll,{push:volantis.scroll.engine.push}),volantis.scroll.getScrollTop=()=>{let e;return window.pageYOffset?e=window.pageYOffset:document.compatMode&&"BackCompat"!=document.compatMode?e=document.documentElement.scrollTop:document.body&&(e=document.body.scrollTop),e},volantis.scroll.scrollHeight=function(){return Math.max(document.body.scrollHeight,document.documentElement.scrollHeight)},volantis.scroll.offsetHeight=function(){return Math.max(document.body.offsetHeight,document.documentElement.offsetHeight,document.body.clientHeight,document.documentElement.clientHeight)},volantis.scroll.progress=function(){return volantis.scroll.getScrollTop()/(volantis.scroll.scrollHeight()-volantis.scroll.offsetHeight())},volantis.scroll.handleScrollEvents=()=>{volantis.scroll.lastScrollTop=volantis.scroll.getScrollTop(),volantis.requestAnimationFrame((function e(){const t=volantis.scroll.getScrollTop();volantis.scroll.lastScrollTop!==t?(volantis.scroll.del=t-volantis.scroll.lastScrollTop,volantis.scroll.lastScrollTop=t,volantis.scroll.unengine.list=[],volantis.scroll.engine.start()):volantis.scroll.unengine.start(),volantis.requestAnimationFrame(e)}))},volantis.scroll.handleScrollEvents(),volantis.scroll.ele=null,volantis.scroll.to=(e,t={})=>{e&&(volantis.scroll.ele=e,opt={top:e.getBoundingClientRect().top+document.documentElement.scrollTop,behavior:"smooth"},"top"in t&&(opt.top=t.top),"behavior"in t&&(opt.behavior=t.behavior),"addTop"in t&&(opt.top+=t.addTop),"observerDic"in t||(t.observerDic=100),window.scrollTo(opt),t.observer&&setTimeout((()=>{volantis.scroll.ele==e&&volantis.scroll.unengine.push((()=>{let o=e.getBoundingClientRect().top;o>=-t.observerDic&&o<=t.observerDic||volantis.scroll.to(e,t),volantis.scroll.unengine.remove("unengineObserver")}),"unengineObserver")}),1e3))},volantis.cleanContentVisibility=()=>{document.querySelector(".post-story")&&(console.log("cleanContentVisibility"),document.querySelectorAll(".post-story").forEach((e=>{e.classList.remove("post-story")})))}</script><script>volantis.GLOBAL_CONFIG={root:"/",debug:!1,default:{avatar:"https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/placeholder/avatar/round/3442075.svg",link:"https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/placeholder/link/8f277b4ee0ecd.svg",cover:"https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/placeholder/cover/76b86c0226ffd.svg",image:"https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/placeholder/image/2659360.svg"},lastupdate:new Date(1739832285375),cdn:{izitoast_css:"https://cdn.bootcdn.net/ajax/libs/izitoast/1.4.0/css/iziToast.min.css",izitoast_js:"https://cdn.bootcdn.net/ajax/libs/izitoast/1.4.0/js/iziToast.min.js",fancybox_css:"https://cdn.bootcdn.net/ajax/libs/fancyapps-ui/4.0.31/fancybox.min.css",fancybox_js:"https://cdn.bootcdn.net/ajax/libs/fancyapps-ui/4.0.31/fancybox.umd.min.js"},sidebar:{for_page:["blogger","navigation","webinfo"],for_post:["toc"],webinfo:{lastupd:{enable:!1,friendlyShow:!0},runtime:{data:"2019/08/19",unit:"天"}}},plugins:{message:{enable:!1,icon:{default:"fa-solid fa-info-circle light-blue",quection:"fa-solid fa-question-circle light-blue"},time:{default:5e3,quection:2e4},position:"topRight",transitionIn:"bounceInLeft",transitionOut:"fadeOutRight",titleColor:"var(--color-text)",messageColor:"var(--color-text)",backgroundColor:"var(--color-card)",zindex:2147483647,copyright:{enable:!0,title:"知识共享许可协议",message:"请遵守 CC BY-NC-SA 4.0 协议。",icon:"far fa-copyright light-blue"},aplayer:{enable:!0,play:"fa-solid fa-play",pause:"fa-solid fa-pause"},rightmenu:{enable:!0,notice:!0}},aplayer:{id:1480000098,enable:!0}},search:{dataPath:("/".endsWith("/")?"/":"//")+"content.json"},languages:{search:{hits_empty:"找不到您查询的内容：${query}",hits_stats:"找到 ${hits} 条结果，用时 ${time} 毫秒"}}}</script></head><body itemscope itemtype="http://schema.org/WebPage"><header itemscope itemtype="http://schema.org/WPHeader" id="l_header" class="l_header auto shadow floatable blur show" style="opacity:0"><div class="container"><div id="wrapper"><div class="nav-sub"><p class="title"></p><ul class="switcher nav-list-h m-phone" id="pjax-header-nav-list"><li><a id="s-comment" class="fa-solid fa-comments fa-fw" target="_self" title="comment"></a></li><li><a id="s-toc" class="s-toc fa-solid fa-list fa-fw" target="_self" title="toc"></a></li></ul></div><div class="nav-main"> <a class="title flat-box" target="_self" href="/">MHuiG's Blog</a><div class="menu navigation"><ul class="nav-list-h m-pc"><li><a class="menuitem flat-box faa-parent animated-hover" title="索引"><i class="fa-duotone fa-list-alt fa-fw"></i> 索引</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" href="/notes/" title="便签" active-action="action-notes"><i class="fa-duotone fa-books fa-fw"></i> 便签</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/categories/" title="分类" active-action="action-categories"><i class="fa-duotone fa-folder-open fa-fw"></i> 分类</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/tags/" title="标签" active-action="action-tags"><i class="fa-duotone fa-tags fa-fw"></i> 标签</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/archives/" title="归档" active-action="action-archives"><i class="fa-duotone fa-archive fa-fw"></i> 归档</a></li></ul></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/friends/" title="友链" active-action="action-pagesfriends"><i class="fa-duotone fa-link fa-fw"></i> 友链</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" href="https://www.travellings.cn/go-by-clouds.html" title="Travelling" target="_blank" active-action="action-https:wwwtravellingscngo-by-cloudshtml" rel="external nofollow noopener noreferrer"><i class="fa-duotone fa-subway fa-fw"></i> Travelling</a></li></ul></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/about/" title="关于" active-action="action-pagesabout"><i class="fa-duotone fa-user-tie fa-fw"></i> 关于</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="noopener" href="https://mhuig.top/privacy-policy/" title="隐私政策" active-action="action-https:mhuigtopprivacy-policy"><i class="fa-duotone fa-user-secret fa-fw"></i> 隐私政策</a></li></ul></li><li><a class="menuitem flat-box faa-parent animated-hover" title="更多"><i class="fa-duotone fa-fan fa-spin fa-fw"></i> 更多</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/beer/" title="Sponsor" active-action="action-pagesbeer"><i class="fa-duotone fa-heart-square fa-fw"></i> Sponsor</a></li><hr><li><a class="menuitem flat-box faa-parent animated-hover" title="博客管理"><i class="fa-duotone fa-user-shield fa-fw"></i> 博客管理</a><ul class="list-v"><li><a class="menuitem flat-box header toggle-mode-btn"><i class="fa-duotone fa-bat fa-fw"></i> 暗黑模式</a></li><li></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/rss/" title="RSS订阅" active-action="action-pagesrss"><i class="fa-duotone fa-rss fa-fw"></i> RSS订阅</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/sitemap.xml" title="站点地图" active-action="action-sitemapxml"><i class="fa-duotone fa-sitemap fa-fw"></i> 站点地图</a></li><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://mhuig.instatus.com/" title="Monitors" active-action="action-https:mhuiginstatuscom"><i class="fa-duotone fa-telescope fa-fw"></i> Monitors</a></li><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://ssl.mhuig.top/" title="SSL Status" active-action="action-https:sslmhuigtop"><i class="fa-brands fa-expeditedssl fa-fw"></i> SSL Status</a></li></ul></li><hr><li><a class="menuitem flat-box faa-parent animated-hover" title="工具箱"><i class="fa-duotone fa-tools fa-fw"></i> 工具箱</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://rssbox.mhuig.top/" title="RSS Box" active-action="action-https:rssboxmhuigtop"><i class="fa-duotone fa-conveyor-belt-boxes fa-fw"></i> RSS Box</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/havefun/Coco/" title="My Cat" active-action="action-havefunCoco"><i class="fa-duotone fa-cat fa-fw"></i> My Cat</a></li><li><a class="menuitem flat-box"><i class="fa-duotone fa-compact-disc fa-spin fa-fw music"></i> Music</a><ul class="list-v"><li><div class="aplayer-container"><div class="aplayer-local"></div></div></li></ul></li><li></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/talk/" title="碎言碎语" active-action="action-pagestalk"><i class="fa-duotone fa-comments fa-fw"></i> 碎言碎语</a></li></ul></li><hr><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://github.com/MHuiG" title="GitHub" active-action="action-https:githubcomMHuiG"><i class="fa-brands fa-github fa-fw"></i> GitHub</a></li></ul></li></ul></div><div class="m_search"><form name="searchform" class="form u-search-form"><i class="icon fa-solid fa-search fa-fw"></i> <input type="text" class="input u-search-input" placeholder="Search..."></form></div><ul class="switcher nav-list-h m-phone"><li><a class="s-search fa-solid fa-search fa-fw" target="_self" title="search"></a></li><li><a class="s-menu fa-solid fa-bars fa-fw" target="_self" title="menu"></a><ul class="menu-phone list-v navigation white-box"><li><a class="menuitem flat-box faa-parent animated-hover" title="索引"><i class="fa-duotone fa-list-alt fa-fw"></i> 索引</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" href="/notes/" title="便签" active-action="action-notes"><i class="fa-duotone fa-books fa-fw"></i> 便签</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/categories/" title="分类" active-action="action-categories"><i class="fa-duotone fa-folder-open fa-fw"></i> 分类</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/tags/" title="标签" active-action="action-tags"><i class="fa-duotone fa-tags fa-fw"></i> 标签</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/archives/" title="归档" active-action="action-archives"><i class="fa-duotone fa-archive fa-fw"></i> 归档</a></li></ul></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/friends/" title="友链" active-action="action-pagesfriends"><i class="fa-duotone fa-link fa-fw"></i> 友链</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" href="https://www.travellings.cn/go-by-clouds.html" title="Travelling" target="_blank" active-action="action-https:wwwtravellingscngo-by-cloudshtml" rel="external nofollow noopener noreferrer"><i class="fa-duotone fa-subway fa-fw"></i> Travelling</a></li></ul></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/about/" title="关于" active-action="action-pagesabout"><i class="fa-duotone fa-user-tie fa-fw"></i> 关于</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="noopener" href="https://mhuig.top/privacy-policy/" title="隐私政策" active-action="action-https:mhuigtopprivacy-policy"><i class="fa-duotone fa-user-secret fa-fw"></i> 隐私政策</a></li></ul></li><li><a class="menuitem flat-box faa-parent animated-hover" title="更多"><i class="fa-duotone fa-fan fa-spin fa-fw"></i> 更多</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/beer/" title="Sponsor" active-action="action-pagesbeer"><i class="fa-duotone fa-heart-square fa-fw"></i> Sponsor</a></li><hr><li><a class="menuitem flat-box faa-parent animated-hover" title="博客管理"><i class="fa-duotone fa-user-shield fa-fw"></i> 博客管理</a><ul class="list-v"><li><a class="menuitem flat-box header toggle-mode-btn"><i class="fa-duotone fa-bat fa-fw"></i> 暗黑模式</a></li><li></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/rss/" title="RSS订阅" active-action="action-pagesrss"><i class="fa-duotone fa-rss fa-fw"></i> RSS订阅</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/sitemap.xml" title="站点地图" active-action="action-sitemapxml"><i class="fa-duotone fa-sitemap fa-fw"></i> 站点地图</a></li><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://mhuig.instatus.com/" title="Monitors" active-action="action-https:mhuiginstatuscom"><i class="fa-duotone fa-telescope fa-fw"></i> Monitors</a></li><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://ssl.mhuig.top/" title="SSL Status" active-action="action-https:sslmhuigtop"><i class="fa-brands fa-expeditedssl fa-fw"></i> SSL Status</a></li></ul></li><hr><li><a class="menuitem flat-box faa-parent animated-hover" title="工具箱"><i class="fa-duotone fa-tools fa-fw"></i> 工具箱</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://rssbox.mhuig.top/" title="RSS Box" active-action="action-https:rssboxmhuigtop"><i class="fa-duotone fa-conveyor-belt-boxes fa-fw"></i> RSS Box</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/havefun/Coco/" title="My Cat" active-action="action-havefunCoco"><i class="fa-duotone fa-cat fa-fw"></i> My Cat</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/talk/" title="碎言碎语" active-action="action-pagestalk"><i class="fa-duotone fa-comments fa-fw"></i> 碎言碎语</a></li></ul></li><hr><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://github.com/MHuiG" title="GitHub" active-action="action-https:githubcomMHuiG"><i class="fa-brands fa-github fa-fw"></i> GitHub</a></li></ul></li></ul></li></ul></div></div></div></header><div id="l_body"><div id="l_cover"><div id="none" class="cover-wrapper post featured" style="display:none"><div class="cover-bg lazyload placeholder" data-bg></div><div class="cover-body"><div class="top"><p class="title">MHuiG</p><p class="subtitle">「看庭前花开花落,望天上云卷云舒」</p></div><div class="bottom"><div class="menu navigation"><div class="list-h"><a target="_blank" rel="external nofollow noopener noreferrer" href="https://github.com/MHuiG" active-action="action-https:githubcomMHuiG"><i class="fa-brands fa-github color-github fa-fw"></i><p>Github</p></a><a href="/pages/friends/" active-action="action-pagesfriends"><i class="fa-duotone fa-link color-friends fa-fw"></i><p>友链</p></a><a href="/pages/about/" active-action="action-pagesabout"><i class="fa-duotone fa-user-tie color-about fa-fw"></i><p>关于</p></a><a href="https://www.travellings.cn/go-by-clouds.html" target="_blank" active-action="action-https:wwwtravellingscngo-by-cloudshtml" rel="external nofollow noopener noreferrer"><i class="fa-duotone fa-subway color-travellings fa-fw"></i><p>Travelling</p></a></div></div></div></div><div id="scroll-down" style="display:none"><i class="fa fa-chevron-down scroll-down-effects"></i></div></div></div><div id="safearea"><div class="body-wrapper"><div id="l_main" class><article itemscope itemtype="http://schema.org/Article" class="article post white-box reveal md shadow floatable blur article-type-post" id="post" itemprop="blogPost"><link itemprop="mainEntityOfPage" href="https://blog.mhuig.top/p/48f700eb/"><span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization"><meta itemprop="name" content="MHuiG"></span><span hidden itemprop="post" itemscope itemtype="http://schema.org/Post"><meta itemprop="name" content="MHuiG"><meta itemprop="description" content="MHuiG&#39;s Blog (MHuiG的博客) MHuiG&#39;s Neverland（MHuiG的梦幻岛） —— MHuiG(@MHuiG) 随便写写画画的地方 - 技术博客"></span><div class="headimg-div"> <a class="headimg-a"><img itemprop="image" class="headimg lazyload" src="https://static.mhuig.top/npm/mhgoos@0.0.1651974150318/2020116102430.webp" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651974150318/2020116102430.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=="></a></div><div class="article-meta" id="top"><h1 class="title" itemprop="name headline"> BERT 预训练模型及其应用案例</h1><div class="new-meta-box"><div class="new-meta-item author" itemprop="author" itemscope itemtype="http://schema.org/Person"> <a itemprop="url" class="author" target="_blank" href="https://mhuig.top/" rel="nofollow noopener"><img itemprop="image" src="/lib/avatar/avatar-16.webp?v=2199236952" class="lazyload" data-srcset="/lib/avatar/avatar-16.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=="><p itemprop="name">MHuiG</p></a></div><div class="new-meta-item wordcount"><a class="notlink"><i class="fa-duotone fa-keyboard fa-fw" aria-hidden="true"></i><p>字数：6.7k 字</p></a></div><div class="new-meta-item readtime"><a class="notlink"><i class="fa-duotone fa-hourglass-half fa-fw" aria-hidden="true"></i><p>时长：25 分钟</p></a></div></div></div><div id="layoutHelper-page-plugins"></div><div id="post-body" itemprop="articleBody"><p>预训练模型最开始是在图像领域提出的，获得了良好的效果，近几年才被广泛应用到自然语言处理各项任务中。</p><ul><li><p>(1) 2003 年 Bengio 提出神经网络语言模型 NNLM，从此统一了 NLP 的特征形式 ——Embedding；</p></li><li><p>(2) 2013 年 Mikolov 提出词向量 Word2vec，延续 NNLM 又引入了大规模预训练（Pretrain）的思路；</p></li><li><p>(3) 2017 年 Vaswani 提出 Transformer 模型，实现用一个模型处理多种 NLP 任务。</p></li><li><p>(4) 基于 Transformer 架构，2018 年底开始出现一大批预训练语言模型 (3 个预训练代表性模型 BERT [2018]、XLNet [2019] 和 MPNet [2020])，刷新众多 NLP 任务，形成新的里程碑事件。</p></li></ul><span id="more"></span><p>预训练模型的应用通常分为两步:</p><ul><li><p>第一步：在计算性能满足的情况下用某个较大的数据集训练出一个较好的模型。</p></li><li><p>第二步：根据不同的任务，改造预训练模型，用新任务的数据集在预训练模型上进行微调。</p></li></ul><p>预训练模型的好处是训练代价较小，配合下游任务可以实现更快的收敛速度，并且能够有效地提高模型性能，尤其是对一些训练数据比较稀缺的任务。换句话说，预训练方法可以认为是让模型基于一个更好的初始状态进行学习，从而能够达到更好的性能。</p><p>要讲自然语言的预训练，得先从图像领域的预训练说起。</p><div class="story post-story"><h2 id="图像领域的预训练"><a href="#图像领域的预训练" class="headerlink" title="图像领域的预训练"></a>图像领域的预训练</h2><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055185325/2020115152532.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055185325/2020115152532.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="图像领域的预训练"></p><p>设计好网络结构以后，对于图像来说一般是 CNN 的多层叠加网络结构，可以先用某个训练集合比如训练集合 A 或者训练集合 B 对这个网络进行预先训练，在 A 任务上或者 B 任务上学会网络参数，然后存起来以备后用。假设我们面临第三个任务 C，网络结构采取相同的网络结构，在比较浅的几层 CNN 结构，网络参数初始化的时候可以加载 A 任务或者 B 任务学习好的参数，其它 CNN 高层参数仍然随机初始化。</p><p>之后我们用 C 任务的训练数据来训练网络，此时有两种做法，一种是浅层加载的参数在训练 C 任务过程中不动，这种方法被称为 “Frozen”;</p><p>另外一种是底层网络参数尽管被初始化了，在 C 任务训练过程中仍然随着训练的进程不断改变，这种一般叫 “Fine-Tuning”，顾名思义，就是更好地把参数进行调整使得更适应当前的 C 任务。</p><p>对于层级的 CNN 结构来说，不同层级的神经元学习到了不同类型的图像特征，由底向上特征形成层级结构。</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055230257/2020115152735.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055230257/2020115152735.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="对于层级的CNN结构来说，不同层级的神经元学习到了不同类型的图像特征，由底向上特征形成层级结构"></p><p> 如果我们手头是个人脸识别任务，训练好网络后，把每层神经元学习到的特征可视化肉眼看一看每层学到了啥特征，你会看到最底层的神经元学到的是线段等特征，图示的第二个隐层学到的是人脸五官的轮廓，第三层学到的是人脸的轮廓，通过三步形成了特征的层级结构，越是底层的特征越是所有不论什么领域的图像都会具备的比如边角线弧线等底层基础特征，越往上抽取出的特征越与手头任务相关。</p><p>正因为此，所以预训练好的网络参数，尤其是底层的网络参数抽取出特征跟具体任务越无关，越具备任务的通用性，所以这是为何一般用底层预训练好的参数初始化新任务网络参数的原因。</p><p>而高层特征跟任务关联较大，实际可以不用使用，或者采用 Fine-tuning 用新数据集合清洗掉高层无关的特征抽取器。</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055412012/2020115152826.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055412012/2020115152826.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="用ImageNet来做网络的预训练"></p><p> 一般我们用 ImageNet 来做网络的预训练，主要有两点，一方面 ImageNet 是图像领域里有超多事先标注好训练数据的数据集合，分量足是个很大的优势，量越大训练出的参数越靠谱；另外一方面因为 ImageNet 有 1000 类，类别多，算是通用的图像数据，跟领域没太大关系，所以通用性好。</p></div><div class="story post-story"><h2 id="Word-Embedding"><a href="#Word-Embedding" class="headerlink" title="Word Embedding"></a>Word Embedding</h2><p>现有的机器学习方法往往无法直接处理文本数据，因此需要找到合适的方法，将文本数据转换为数值型数据，由此引出了 Word Embedding 的概念，Word Embedding 算法携带了语义信息且维度经过压缩便于运算。</p><h3 id="语言模型"><a href="#语言模型" class="headerlink" title="语言模型"></a>语言模型</h3><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055460011/2020115152917.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055460011/2020115152917.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="语言模型"></p><p>为了能够量化地衡量哪个句子更像一句人话，可以设计如上图所示函数，核心函数 P 的思想是根据句子里面前面的一系列前导单词预测后面单词的概率大小。</p><h3 id="神经网络语言模型"><a href="#神经网络语言模型" class="headerlink" title="神经网络语言模型"></a>神经网络语言模型</h3><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055522139/2020115152938.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055522139/2020115152938.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="神经网络语言模型"></p><p>NNLM 是从语言模型出发 (即计算概率角度)，构建神经网络针对目标函数对模型进行最优化，训练的起点是使用神经网络去搭建语言模型实现词的预测任务，并且在优化过程后模型的副产品就是词向量。</p><h3 id="Word2Vec"><a href="#Word2Vec" class="headerlink" title="Word2Vec"></a>Word2Vec</h3><p>2013 年最火的用语言模型做 Word Embedding 的工具是 Word2Vec，后来又出了 Glove。</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055586877/2020115152959.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055586877/2020115152959.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="Word2Vec"></p><p>Word2Vec 有两种训练方法，一种叫 CBOW，核心思想是从一个句子里面把一个词抠掉，用这个词的上文和下文去预测被抠掉的这个词；</p><p>第二种叫做 Skip-gram，和 CBOW 正好反过来，输入某个单词，要求网络预测它的上下文单词。</p><p>使用 Word2Vec 或者 Glove，通过做语言模型任务，就可以获得每个单词的 Word Embedding。</p><h3 id="Word-Embedding的使用"><a href="#Word-Embedding的使用" class="headerlink" title="Word Embedding的使用"></a>Word Embedding 的使用</h3><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055630118/2020115153022.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055630118/2020115153022.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="Word Embedding的使用"></p><p>我们有个 NLP 的下游任务，比如 QA，就是问答问题，所谓问答问题，指的是给定一个问题 X，给定另外一个句子 Y, 要判断句子 Y 是否是问题 X 的正确答案。</p><p>句子中每个单词以 Onehot 形式作为输入，然后乘以 Word Embedding 矩阵 Q，就直接取出单词对应的 Word Embedding。</p><p>使用 Word Embedding 等价于把 Onehot 层到 embedding 层的网络用预训练好的参数矩阵 Q 初始化。</p><p>这跟前面讲的图像领域的低层预训练过程其实是一样的，区别无非 Word Embedding 只能初始化第一层网络参数，再高层的参数就无能为力了。</p><p>下游 NLP 任务在使用 Word Embedding 的时候也类似图像有两种做法，一种是 Frozen，就是 Word Embedding 那层网络参数固定不动；另外一种是 Fine-Tuning，就是使用新的训练集合训练，在训练过程中，更新 Word Embedding 这层参数。</p><h3 id="Word-Embedding的问题"><a href="#Word-Embedding的问题" class="headerlink" title="Word Embedding的问题"></a>Word Embedding 的问题</h3><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055660434/2020115153044.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055660434/2020115153044.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="Word Embedding的问题"></p><p>是多义词问题。多义词是自然语言中经常出现的现象，也是语言灵活性和高效性的一种体现。</p><p>多义词对 Word Embedding 来说有什么负面影响？如上图所示，比如多义词 Bank，有两个常用含义，但是 Word Embedding 在对 bank 这个单词进行编码的时候，是区分不开这两个含义的，因为它们尽管上下文环境中出现的单词不同，但是在用语言模型训练的时候，不论什么上下文的句子经过 word2vec，都是预测相同的单词 bank，而同一个单词占的是同一行的参数空间，这导致两种不同的上下文信息都会编码到相同的 word embedding 空间里去。所以 word embedding 无法区分多义词的不同语义，这就是它的一个比较严重的问题。</p></div><div class="story post-story"><h2 id="从Word-Embedding到ELMO"><a href="#从Word-Embedding到ELMO" class="headerlink" title="从Word Embedding到ELMO"></a>从 Word Embedding 到 ELMO</h2><p>ELMO 是 “Embedding from Language Models” 的简称。在此之前的 Word Embedding 本质上是个静态的方式，所谓静态指的是训练好之后每个单词的表达就固定住了，以后使用的时候，不论新句子上下文单词是什么，这个单词的 Word Embedding 不会跟着上下文场景的变化而改变。</p><p>ELMO 的本质思想是：事先用语言模型学好一个单词的 Word Embedding，此时多义词无法区分，实际使用 Word Embedding 的时候，单词已经具备了特定的上下文了，这个时候可以根据上下文单词的语义去调整单词的 Word Embedding 表示，这样经过调整后的 Word Embedding 更能表达在这个上下文中的具体含义，自然也就解决了多义词的问题了。所以 ELMO 本身的思路是根据当前上下文对 Word Embedding 动态调整。</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055689969/202011515317.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055689969/202011515317.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="从Word Embedding到ELMO"></p><p>ELMO 采用了典型的两阶段过程，第一个阶段是利用语言模型进行预训练；第二个阶段是在做下游任务时，从预训练网络中提取对应单词的网络各层的 Word Embedding 作为新特征补充到下游任务中。</p><p>使用这个网络结构利用大量语料做语言模型任务就能预先训练好这个网络，如果训练好这个网络后，输入一个新句子 ，句子中每个单词都能得到对应的三个 Embedding: 最底层是单词的 Word Embedding，往上走是第一层双向 LSTM 中对应单词位置的 Embedding，这层编码单词的句法信息更多一些；再往上走是第二层 LSTM 中对应单词位置的 Embedding，这层编码单词的语义信息更多一些。也就是说，ELMO 的预训练过程不仅仅学会单词的 Word Embedding，还学会了一个双层双向的 LSTM 网络结构，而这两者后面都有用。</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055719114/2020115153133.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055719114/2020115153133.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="从Word Embedding到ELMO"></p><p> 上图展示了下游任务的使用过程，比如我们的下游任务仍然是 QA 问题，此时对于问句 X，我们可以先将句子 X 作为预训练好的 ELMO 网络的输入，这样句子 X 中每个单词在 ELMO 网络中都能获得对应的三个 Embedding，之后给予这三个 Embedding 中的每一个 Embedding 一个权重 a，这个权重可以学习得来，根据各自权重累加求和，将三个 Embedding 整合成一个。然后将整合后的这个 Embedding 作为 X 句在自己任务的那个网络结构中对应单词的输入，以此作为补充的新特征给下游任务使用。对于上图所示下游任务 QA 中的回答句子 Y 来说也是如此处理。因为 ELMO 给下游提供的是每个单词的特征形式，所以这一类预训练的方法被称为 “Feature-based Pre-Training”。</p></div><div class="story post-story"><h2 id="从Word-Embedding到GPT"><a href="#从Word-Embedding到GPT" class="headerlink" title="从Word Embedding到GPT"></a>从 Word Embedding 到 GPT</h2><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055747746/2020115153159.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055747746/2020115153159.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="从Word Embedding到GPT"></p><p>GPT 是 “Generative Pre-Training” 的简称，从名字看其含义是指的生成式的预训练。GPT 也采用两阶段过程，第一个阶段是利用语言模型进行预训练，第二阶段通过 Fine-tuning 的模式解决下游任务。</p><h3 id="Transformer"><a href="#Transformer" class="headerlink" title="Transformer"></a>Transformer</h3><p>Transformer 是个叠加的 “自注意力机制（Self Attention）” 构成的深度网络，是目前 NLP 里最强的特征提取器。</p><p>Transformer 是一种基于 encoder-decoder 结构的模型。在机器翻译任务上的表现超过了 RNN，CNN，只用 encoder-decoder 和 attention 机制就能达到很好的效果，最大的优点是可以高效地并行化。</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055774401/2020115153231.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055774401/2020115153231.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="Transformer"></p><h3 id="自注意力机制模型"><a href="#自注意力机制模型" class="headerlink" title="自注意力机制模型"></a>自注意力机制模型</h3><p>人类视觉通过快速扫描全局图像，获得需要重点关注的目标区域，也就是一般所说的注意力焦点，而后对这一区域投入更多注意力资源，以获取更多所需要关注目标的细节信息，而抑制其他无用信息。</p><p>这是人类利用有限的注意力资源从大量信息中快速筛选出高价值信息的手段.</p><p>深度学习中的注意力机制从本质上讲和人类的选择性视觉注意力机制类似，核心目标也是从众多信息中选择出对当前任务目标更关键的信息。</p><p>Attention 在同一个英语句子内单词间产生的联系。</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055805502/202011515330.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055805502/202011515330.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="自注意力机制模型"></p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055832227/2020115153319.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055832227/2020115153319.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="自注意力机制模型"></p><p>Self Attention 可以捕获同一个句子中单词之间的一些句法特征（比如图展示的有一定距离的短语结构）或者语义特征（比如图展示的 its 的指代对象 Law）。</p><p>很明显，引入 Self Attention 后会更容易捕获句子中长距离的相互依赖的特征，因为如果是 RNN 或者 LSTM，需要依次序序列计算，对于远距离的相互依赖的特征，要经过若干时间步步骤的信息累积才能将两者联系起来，而距离越远，有效捕获的可能性越小。</p><p>SelfAttention 在计算过程中会直接将句子中任意两个单词的联系通过一个计算步骤直接联系起来，所以远距离依赖特征之间的距离被极大缩短，有利于有效地利用这些特征。除此外，SelfAttention 对于增加计算的并行性也有直接帮助作用。这是为何 Self Attention 逐渐被广泛使用的主要原因。</p><h3 id="GPT如何使用"><a href="#GPT如何使用" class="headerlink" title="GPT如何使用"></a>GPT 如何使用</h3><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055862819/2020115153347.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055862819/2020115153347.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="GPT如何使用"></p><p>把任务的网络结构改造成和 GPT 的网络结构是一样的。然后，在做下游任务的时候，利用第一步预训练好的参数初始化 GPT 的网络结构，对网络参数进行 Fine-tuning，使得这个网络更适合解决手头的问题。</p></div><div class="story post-story"><h2 id="从GPT和ELMO及word2Vec到Bert"><a href="#从GPT和ELMO及word2Vec到Bert" class="headerlink" title="从GPT和ELMO及word2Vec到Bert"></a>从 GPT 和 ELMO 及 word2Vec 到 Bert</h2><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055892617/2020115153419.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055892617/2020115153419.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="从GPT和ELMO及word2Vec到Bert"></p><p>Bert 采用和 GPT 完全相同的两阶段模型，首先是语言模型预训练；其次是使用 Fine-Tuning 模式解决下游任务。和 GPT 的最主要不同在于在预训练阶段采用了类似 ELMO 的双向语言模型，当然另外一点是语言模型的数据规模要比 GPT 大。</p><p>BERT 本质上是一个自编码（Auto Encoder）语言模型，为了能见多识广，BERT 使用 3 亿多词语训练，采用 12 层双向 Transformer 架构。注意，BERT 只使用了 Transformer 的编码器部分，可以理解为 BERT 旨在学习庞大文本的内部语义信息。</p><p>具体训练目标之一，是被称为掩码语言模型的 MLM。即输入一句话，给其中 15% 的字打上 “mask” 标记，经过 Embedding 输入和 12 层 Transformer 深度理解，来预测 “mask” 标记的地方原本是哪个字。</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="built_in">input</span>: 欲把西[mask]比西子，淡[mask]浓抹总相宜</span><br><span class="line">output: 欲把西[湖]比西子，淡[妆]浓抹总相宜</span><br></pre></td></tr></tbody></table></figure><p>例如我们输入 “欲把西 [mask] 比西子，淡 [mask] 浓抹总相宜” 给 BERT，它需要根据没有被 “mask” 的上下文，预测出掩盖的地方是 “湖” 和 “妆”。</p><p>MLM 任务的灵感来自于人类做完形填空。挖去文章中的某些片段，需要通过上下文理解来猜测这些被掩盖位置原先的内容。</p><p>训练目标之二，是预测输入的两句话之间是否为上下文（NSP）的二分类问题。继续输入 “欲把西 [湖] 比西子，淡 [妆] 浓抹总相宜”，BERT 将预测这两句话的组合是否合理（这个例子是 “yes”）。（随后的研究者对预训练模型探索中证明，NSP 任务过于简单，对语言模型的训练作用并不是很大）</p><p>通过这两个任务和大规模语料训练，BERT 语言模型可以很好学习到文本之间的蕴含的关系。</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055923050/2020115153516.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055923050/2020115153516.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="从GPT和ELMO及word2Vec到Bert"></p></div><div class="story post-story"><h2 id="NLP的四大任务"><a href="#NLP的四大任务" class="headerlink" title="NLP的四大任务"></a>NLP 的四大任务</h2><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055950986/2020115153626.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055950986/2020115153626.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="NLP的四大任务"></p><p>绝大部分 NLP 问题可以归入上图所示的四类任务中：</p><p>一类是序列标注，这是最典型的 NLP 任务，比如中文分词，词性标注，命名实体识别，语义角色标注等都可以归入这一类问题，它的特点是句子中每个单词要求模型根据上下文都要给出一个分类类别。</p><p>第二类是分类任务，比如我们常见的文本分类，情感计算等都可以归入这一类。它的特点是不管文章有多长，总体给出一个分类类别即可。</p><p>第三类任务是句子关系判断，比如 Entailment，QA，语义改写，自然语言推理等任务都是这个模式，它的特点是给定两个句子，模型判断出两个句子是否具备某种语义关系；</p><p>第四类是生成式任务，比如机器翻译，文本摘要，写诗造句，看图说话等都属于这一类。它的特点是输入文本内容后，需要自主生成另外一段文字。</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652055979818/2020115153650.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652055979818/2020115153650.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="NLP的四大任务"></p><p>根据任务选择不同的预训练数据初始化 encoder 和 decoder 即可。这是相当直观的一种改造方法。当然，也可以更简单一点，比如直接在单个 Transformer 结构上加装隐层产生输出也是可以的。不论如何，从这里可以看出，NLP 四大类任务都可以比较方便地改造成 Bert 能够接受的方式。这其实是 Bert 的非常大的优点，这意味着它几乎可以做任何 NLP 的下游任务，具备普适性，这是很强的。</p></div><div class="story post-story"><h2 id="BERT的应用案例"><a href="#BERT的应用案例" class="headerlink" title="BERT的应用案例"></a>BERT 的应用案例</h2><h3 id="下载bert预训练模型"><a href="#下载bert预训练模型" class="headerlink" title="下载bert预训练模型"></a>下载 bert 预训练模型</h3><p>Google - BERT 源码 <a target="_blank" rel="external nofollow noopener noreferrer" href="https://github.com/google-research/bert%E4%B8%8B%E8%BD%BD%E9%A2%84%E8%AE%AD%E7%BB%83%E6%A8%A1%E5%9E%8B%E3%80%82">https://github.com/google-research/bert 下载预训练模型。</a></p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056086874/2020115153710.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056086874/2020115153710.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="下载bert预训练模型"></p><p>我这里选择中文的 BERT，下载解压后的目录如下：</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056114238/2020115153734.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056114238/2020115153734.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="下载bert预训练模型"></p><h3 id="安装bert-as-service"><a href="#安装bert-as-service" class="headerlink" title="安装bert-as-service"></a>安装 bert-as-service</h3><p>顾名思义，将 BERT 模型直接封装成一个服务，堪称上手最快的 BERT 工具。作者是肖涵博士。</p><p>使用 pip 安装：</p><figure class="highlight shell"><table><tbody><tr><td class="code"><pre><span class="line">pip install bert-serving-server # server</span><br><span class="line">pip install bert-serving-client # client, independent of `bert-serving-server</span><br></pre></td></tr></tbody></table></figure><p>开启 BERT service</p><figure class="highlight shell"><table><tbody><tr><td class="code"><pre><span class="line">bert-serving-start -model_dir E:\nlp\chinese_L-12_H-768_A-12</span><br></pre></td></tr></tbody></table></figure><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056156837/2020115153752.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056156837/2020115153752.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="安装bert-as-service"></p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056192922/2020115153813.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056192922/2020115153813.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="安装bert-as-service"></p><p>使用客户端获取句子编码</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056221610/2020115153830.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056221610/2020115153830.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="安装bert-as-service"></p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056253075/2020115153851.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056253075/2020115153851.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="安装bert-as-service"></p><h3 id="案例一-查找最相近的句子"><a href="#案例一-查找最相近的句子" class="headerlink" title="案例一 查找最相近的句子"></a>案例一 查找最相近的句子</h3><p>根据 bert 获取句子向量，并计算出句子之间的余弦相似度，找出最相似的句子。</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="comment"># 导入bert客户端</span></span><br><span class="line"><span class="keyword">from</span> bert_serving.client <span class="keyword">import</span> BertClient</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">SimilarModel</span>:</span><br><span class="line"> <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line"> self.bert_client = BertClient()</span><br><span class="line"></span><br><span class="line"> <span class="keyword">def</span> <span class="title function_">close_bert</span>(<span class="params">self</span>):</span><br><span class="line"> self.bert_client .close()</span><br><span class="line"></span><br><span class="line"> <span class="keyword">def</span> <span class="title function_">get_sentence_vec</span>(<span class="params">self,sentence</span>):</span><br><span class="line"> <span class="string">'''</span></span><br><span class="line"><span class="string"> 根据bert获取句子向量</span></span><br><span class="line"><span class="string"> :param sentence:</span></span><br><span class="line"><span class="string"> :return:</span></span><br><span class="line"><span class="string"> '''</span></span><br><span class="line"> <span class="keyword">return</span> self.bert_client .encode([sentence])[<span class="number">0</span>]</span><br><span class="line"></span><br><span class="line"> <span class="keyword">def</span> <span class="title function_">cos_similar</span>(<span class="params">self,sen_a_vec, sen_b_vec</span>):</span><br><span class="line"> <span class="string">'''</span></span><br><span class="line"><span class="string"> 计算两个句子的余弦相似度</span></span><br><span class="line"><span class="string"> :param sen_a_vec:</span></span><br><span class="line"><span class="string"> :param sen_b_vec:</span></span><br><span class="line"><span class="string"> :return:</span></span><br><span class="line"><span class="string"> '''</span></span><br><span class="line"> vector_a = np.mat(sen_a_vec)</span><br><span class="line"> vector_b = np.mat(sen_b_vec)</span><br><span class="line"> num = <span class="built_in">float</span>(vector_a * vector_b.T)</span><br><span class="line"> denom = np.linalg.norm(vector_a) * np.linalg.norm(vector_b)</span><br><span class="line"> cos = num / denom</span><br><span class="line"> <span class="keyword">return</span> cos</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__==<span class="string">'__main__'</span>:</span><br><span class="line"> <span class="comment"># 从候选集condinates 中选出与sentence_a 最相近的句子</span></span><br><span class="line"> condinates = [<span class="string">'为什么天空是蔚蓝色的'</span>,<span class="string">'太空为什么是黑的？'</span>,<span class="string">'天空怎么是蓝色的'</span>,<span class="string">'明天去爬山如何'</span>]</span><br><span class="line"> sentence_a = <span class="string">'天空为什么是蓝色的'</span></span><br><span class="line"> bert_client = SimilarModel()</span><br><span class="line"> max_cos_similar = <span class="number">0</span></span><br><span class="line"> most_similar_sentence = <span class="string">''</span></span><br><span class="line"> <span class="keyword">for</span> sentence_b <span class="keyword">in</span> condinates:</span><br><span class="line"> sentence_a_vec = bert_client.get_sentence_vec(sentence_a)</span><br><span class="line"> sentence_b_vec = bert_client.get_sentence_vec(sentence_b)</span><br><span class="line"> cos_similar = bert_client.cos_similar(sentence_a_vec,sentence_b_vec)</span><br><span class="line"> <span class="keyword">if</span> cos_similar > max_cos_similar:</span><br><span class="line"> max_cos_similar = cos_similar</span><br><span class="line"> most_similar_sentence = sentence_b</span><br><span class="line"></span><br><span class="line"> <span class="built_in">print</span>(<span class="string">'最相似的句子：'</span>,most_similar_sentence)</span><br><span class="line"> bert_client .close_bert()</span><br><span class="line"> <span class="comment"># 为什么天空是蔚蓝色的</span></span><br><span class="line"></span><br></pre></td></tr></tbody></table></figure><h3 id="案例二-简单模糊搜索"><a href="#案例二-简单模糊搜索" class="headerlink" title="案例二 简单模糊搜索"></a>案例二 简单模糊搜索</h3><p>将问题编码为向量：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> bert_serving.client <span class="keyword">import</span> BertClient</span><br><span class="line">doc_vecs = bc.encode(questions)</span><br></pre></td></tr></tbody></table></figure><p>最后，我们准备接收新查询并针对现有问题执行简单的 “模糊” 搜索。为此，每次出现新查询时，我们都将其编码为向量，并使用来计算其点积 doc_vecs。将结果递减排序；并返回前 k 个类似的问题，如下所示：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line"> query = <span class="built_in">input</span>(<span class="string">'your question: '</span>)</span><br><span class="line"> query_vec = bc.encode([query])[<span class="number">0</span>]</span><br><span class="line"> <span class="comment"># compute normalized dot product as score</span></span><br><span class="line"> score = np.<span class="built_in">sum</span>(query_vec * doc_vecs, axis=<span class="number">1</span>) / np.linalg.norm(doc_vecs, axis=<span class="number">1</span>)</span><br><span class="line"> topk_idx = np.argsort(score)[::-<span class="number">1</span>][:topk]</span><br><span class="line"> <span class="built_in">print</span>(<span class="string">'top %d questions similar to "%s"'</span> % (topk, query))</span><br><span class="line"> <span class="keyword">for</span> idx <span class="keyword">in</span> topk_idx:</span><br><span class="line"> <span class="built_in">print</span>(<span class="string">'> %s\t%s'</span> % (<span class="string">'%.1f'</span> % score[idx], questions[idx]))</span><br></pre></td></tr></tbody></table></figure><p>现在运行代码并键入查询，查看此搜索引擎如何处理模糊匹配：</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056314880/202011515405.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056314880/202011515405.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="简单模糊搜索"></p><h3 id="案例三-法条推荐"><a href="#案例三-法条推荐" class="headerlink" title="案例三 法条推荐"></a>案例三 法条推荐</h3><h4 id="介绍"><a href="#介绍" class="headerlink" title="介绍"></a>介绍</h4><p>根据刑事法律文书中的案情描述和事实部分，预测本案涉及的相关法条；</p><h4 id="数据说明"><a href="#数据说明" class="headerlink" title="数据说明"></a>数据说明</h4><p>所使用的数据集是来自 “中国裁判文书网” 公开的刑事法律文书，其中每份数据由法律文书中的案情描述和事实部分组成，同时也包括每个案件所涉及的法条、被告人被判的罪名和刑期长短等要素。</p><p>数据集共包括 268 万刑法法律文书，共涉及 202 条罪名，183 条法条，刑期长短包括 0-25 年、无期、死刑。</p><p>数据利用 json 格式储存，每一行为一条数据，每条数据均为一个字典。</p><p>fact: 事实描述</p><p>meta: 标注信息，标注信息中包括:</p><p>criminals: 被告 (数据中均只含一个被告)</p><p>punish_of_money: 罚款 (单位：元)</p><p>accusation: 罪名</p><p>relevant_articles: 相关法条</p><p>term_of_imprisonment: 刑期</p><p>刑期格式 (单位：月)</p><p>death_penalty: 是否死刑</p><p>life_imprisonment: 是否无期</p><p>imprisonment: 有期徒刑刑期</p><p>这里是简单的一条数据展示:</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">{</span><br><span class="line"> <span class="string">"fact"</span>: <span class="string">"2015年11月5日上午，被告人胡某在平湖市乍浦镇的嘉兴市多凌金牛制衣有限公司车间内，与被害人孙某因工作琐事发生口角，后被告人胡某用木制坐垫打伤被害人孙某左腹部。经平湖公安司法鉴定中心鉴定：孙某的左腹部损伤已达重伤二级。"</span>, </span><br><span class="line"> <span class="string">"meta"</span>: </span><br><span class="line"> { </span><br><span class="line"> <span class="string">"relevant_articles"</span>: [<span class="number">234</span>], </span><br><span class="line"> <span class="string">"accusation"</span>: [<span class="string">"故意伤害"</span>], </span><br><span class="line"> <span class="string">"criminals"</span>: [<span class="string">"胡某"</span>], </span><br><span class="line"> <span class="string">"term_of_imprisonment"</span>:</span><br><span class="line"> { </span><br><span class="line"> <span class="string">"death_penalty"</span>: false, </span><br><span class="line"> <span class="string">"imprisonment"</span>: <span class="number">12</span>, </span><br><span class="line"> <span class="string">"life_imprisonment"</span>: false</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></tbody></table></figure><h4 id="实现流程"><a href="#实现流程" class="headerlink" title="实现流程"></a>实现流程</h4><p>进入 <a target="_blank" rel="external nofollow noopener noreferrer" href="https://github.com/thunlp/OpenCLaP">OpenCLaP</a> 下载刑事文书 BERT，并运行 bert-as-service 服务。</p><h5 id="创建一个连接到BertServer的BertClient"><a href="#创建一个连接到BertServer的BertClient" class="headerlink" title="创建一个连接到BertServer的BertClient"></a>创建一个连接到 BertServer 的 BertClient</h5><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> bert_serving.client <span class="keyword">import</span> ConcurrentBertClient</span><br><span class="line">bc = ConcurrentBertClient()</span><br></pre></td></tr></tbody></table></figure><h5 id="获取编码向量和标签"><a href="#获取编码向量和标签" class="headerlink" title="获取编码向量和标签"></a>获取编码向量和标签</h5><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">get_encodes</span>(<span class="params">x</span>):</span><br><span class="line"> <span class="comment"># x 是 batch_size 大小的行 ，每行都是一个json对象</span></span><br><span class="line"> samples = [json.loads(l) <span class="keyword">for</span> l <span class="keyword">in</span> x] <span class="comment"># 一个 batch_size 大小的连续样本</span></span><br><span class="line"> text = [s[<span class="string">'fact'</span>][:<span class="number">50</span>] + s[<span class="string">'fact'</span>][-<span class="number">50</span>:] <span class="keyword">for</span> s <span class="keyword">in</span> samples] <span class="comment"># 获取案情描述和事实部分文字</span></span><br><span class="line"> features = bc.encode(text) <span class="comment"># 使用bert将字符串列表编码为向量列表</span></span><br><span class="line"> <span class="comment"># 随机选择一个标签</span></span><br><span class="line"> labels = [[<span class="built_in">str</span>(random.choice(s[<span class="string">'meta'</span>][<span class="string">'relevant_articles'</span>]))] <span class="keyword">for</span> s <span class="keyword">in</span> samples]</span><br><span class="line"> <span class="keyword">return</span> features, labels</span><br></pre></td></tr></tbody></table></figure><h5 id="构建TensorFlow-DNN-模型的分类器"><a href="#构建TensorFlow-DNN-模型的分类器" class="headerlink" title="构建TensorFlow DNN 模型的分类器"></a>构建 TensorFlow DNN 模型的分类器</h5><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">estimator = DNNClassifier(</span><br><span class="line"> hidden_units=[<span class="number">512</span>], <span class="comment"># 每层隐藏单元的 Iterable 数.所有层都完全连接.</span></span><br><span class="line"> feature_columns=[tf.feature_column.numeric_column(<span class="string">'feature'</span>, shape=(<span class="number">768</span>,))], <span class="comment"># 包含模型使用的所有特征列的iterable.集合中的所有项目都应该是从 _FeatureColumn 派生的类的实例.</span></span><br><span class="line"> n_classes=<span class="built_in">len</span>(laws), <span class="comment"># 标签类的数量.默认为 2,即二进制分类,必须大于1.</span></span><br><span class="line"> config=run_config, <span class="comment"># RunConfig 对象配置运行时设置</span></span><br><span class="line"> label_vocabulary=laws_str, <span class="comment"># 字符串列表,表示可能的标签值.如果给定,标签必须是字符串类型,并且 label_vocabulary 具有任何值.如果没有给出,这意味着标签已经被编码为整数或者在[0,1]内浮动, n_classes=2 ；并且被编码为{0,1,...,n_classes-1}中的整数值,n_classes> 2.如果没有提供词汇表并且标签是字符串,也会出现错误.</span></span><br><span class="line"> optimizer=tf.train.AdamOptimizer(),<span class="comment"># 优化函数</span></span><br><span class="line"> dropout=<span class="number">0.1</span> <span class="comment"># 当不是 None 时,我们将放弃给定坐标的概率.</span></span><br><span class="line"> )</span><br></pre></td></tr></tbody></table></figure><h5 id="训练和评估"><a href="#训练和评估" class="headerlink" title="训练和评估"></a>训练和评估</h5><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="comment"># 输入函数</span></span><br><span class="line">input_fn = <span class="keyword">lambda</span> fp: (tf.data.TextLineDataset(fp) <span class="comment"># TextLineDataset接口提供了一种方法从数据文件中读取。只需要提供文件名（1个或者多个）。这个接口会自动构造一个dataset，类中保存的元素：文中一行，就是一个元素，是string类型的tenser。</span></span><br><span class="line"> <span class="comment"># map将分别对Dataset的每个元素执行一个函数，而apply将立即对整个Dataset执行一个函数</span></span><br><span class="line"> .apply(tf.contrib.data.shuffle_and_repeat(buffer_size=<span class="number">10000</span>)) <span class="comment"># repeat重复和shuffle重排 tf.data.Dataset.repeat 转换会将输入数据重复有限（或无限）次；每次数据重复通常称为一个周期。tf.data.Dataset.shuffle 转换会随机化数据集样本的顺序。</span></span><br><span class="line"> <span class="comment"># 将此数据集的连续元素合并为批。</span></span><br><span class="line"> .batch(batch_size)</span><br><span class="line"> <span class="comment"># tf.py_func()接收的是tensor，然后将其转化为numpy array送入我们自定义的get_encodes函数，最后再将get_encodes函数输出的numpy array转化为tensor返回</span></span><br><span class="line"> .<span class="built_in">map</span>(<span class="keyword">lambda</span> x: tf.py_func(get_encodes, [x], [tf.float32, tf.string], name=<span class="string">'bert_client'</span>),</span><br><span class="line"> num_parallel_calls=num_parallel_calls)</span><br><span class="line"> .<span class="built_in">map</span>(<span class="keyword">lambda</span> x, y: ({<span class="string">'feature'</span>: x}, y))</span><br><span class="line"> .prefetch(<span class="number">20</span>)) <span class="comment"># 创建一个从该数据集中预提取元素的Dataset 大多数数据集输入管道应以调用结束prefetch。这允许在处理当前元素时准备以后的元素。这通常会提高延迟和吞吐量，但以使用额外的内存存储预取元素为代价。</span></span><br><span class="line"><span class="comment"># TrainSpec确定训练的输入数据以及持续时间</span></span><br><span class="line">train_spec = TrainSpec(input_fn=<span class="keyword">lambda</span>: input_fn(train_fp))</span><br><span class="line"><span class="comment"># EvalSpec结合了训练模型的计算和输出的详细信息.计算由计算指标组成,用以判断训练模型的性能.输出将训练好的模型写入外部存储.</span></span><br><span class="line">eval_spec = EvalSpec(input_fn=<span class="keyword">lambda</span>: input_fn(eval_fp), throttle_secs=<span class="number">0</span>) <span class="comment"># 第一次评估发生在throttle_secs秒后</span></span><br><span class="line"><span class="comment"># 训练和评估</span></span><br><span class="line">train_and_evaluate(estimator, train_spec, eval_spec)</span><br></pre></td></tr></tbody></table></figure><h5 id="运行tensorboard可视化训练过程"><a href="#运行tensorboard可视化训练过程" class="headerlink" title="运行tensorboard可视化训练过程"></a>运行 tensorboard 可视化训练过程</h5><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">tensorboard --logdir=law-model</span><br></pre></td></tr></tbody></table></figure><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056346534/2020115153921.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056346534/2020115153921.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="运行tensorboard可视化训练过程"></p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056371895/2020115154036.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056371895/2020115154036.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="运行tensorboard可视化训练过程"></p><h3 id="案例四-互联网新闻情感分析"><a href="#案例四-互联网新闻情感分析" class="headerlink" title="案例四 互联网新闻情感分析"></a>案例四 互联网新闻情感分析</h3><h4 id="介绍-1"><a href="#介绍-1" class="headerlink" title="介绍"></a>介绍</h4><p>对新闻情绪进行分类，0 代表正面情绪、1 代表中性情绪、2 代表负面情绪。</p><h4 id="数据说明-1"><a href="#数据说明-1" class="headerlink" title="数据说明"></a>数据说明</h4><table><thead><tr><th><strong>Field</strong></th><th><strong>Type</strong></th><th><strong>Description</strong></th></tr></thead><tbody><tr><td><strong>id</strong></td><td>String</td><td> 新闻 ID News ID</td></tr><tr><td><strong>text</strong></td><td>String</td><td> 新闻正文内容 Content of news text</td></tr><tr><td><strong>label</strong></td><td>String</td><td> 新闻情感标签 Emotional label in news</td></tr></tbody></table><h4 id="实现流程-1"><a href="#实现流程-1" class="headerlink" title="实现流程"></a>实现流程</h4><h5 id="加载数据集"><a href="#加载数据集" class="headerlink" title="加载数据集"></a>加载数据集</h5><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">load_dataset</span>(<span class="params">filepath</span>):</span><br><span class="line"> dataset_list = []</span><br><span class="line"> f = <span class="built_in">open</span>(filepath, <span class="string">'r'</span>, encoding=<span class="string">'utf-8'</span>)</span><br><span class="line"> r = csv.reader(f)</span><br><span class="line"> <span class="keyword">for</span> item <span class="keyword">in</span> r:</span><br><span class="line"> <span class="keyword">if</span> r.line_num == <span class="number">1</span>:</span><br><span class="line"> <span class="keyword">continue</span></span><br><span class="line"> dataset_list.append(item)</span><br><span class="line"> </span><br><span class="line"> <span class="comment"># 空元素补0</span></span><br><span class="line"> <span class="keyword">for</span> item <span class="keyword">in</span> dataset_list:</span><br><span class="line"> <span class="keyword">if</span> item[<span class="number">1</span>].strip() == <span class="string">''</span>:</span><br><span class="line"> item[<span class="number">1</span>] = <span class="string">'0'</span></span><br><span class="line"></span><br><span class="line"> <span class="keyword">return</span> dataset_list</span><br></pre></td></tr></tbody></table></figure><h5 id="网络类，全连接层"><a href="#网络类，全连接层" class="headerlink" title="网络类，全连接层"></a>网络类，全连接层</h5><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Net</span>(nn.Module):</span><br><span class="line"> <span class="comment"># in_dim=768, out_dim=3</span></span><br><span class="line"> <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, in_dim, out_dim</span>):</span><br><span class="line"> <span class="built_in">super</span>(Net, self).__init__()</span><br><span class="line"></span><br><span class="line"> self.linear1 = nn.Linear(in_dim, <span class="number">500</span>)</span><br><span class="line"> self.linear2 = nn.Linear(<span class="number">500</span>, <span class="number">400</span>)</span><br><span class="line"> self.linear3 = nn.Linear(<span class="number">400</span>, <span class="number">300</span>)</span><br><span class="line"> self.linear4 = nn.Linear(<span class="number">300</span>, <span class="number">200</span>)</span><br><span class="line"> self.linear5 = nn.Linear(<span class="number">200</span>, out_dim)</span><br><span class="line"> </span><br><span class="line"> <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x</span>):</span><br><span class="line"> x = self.linear1(x)</span><br><span class="line"> x = self.linear2(x)</span><br><span class="line"> x = self.linear3(x)</span><br><span class="line"> x = self.linear4(x)</span><br><span class="line"> x = self.linear5(x)</span><br><span class="line"> <span class="keyword">return</span> x</span><br></pre></td></tr></tbody></table></figure><h5 id="计算每个batch的准确率"><a href="#计算每个batch的准确率" class="headerlink" title="计算每个batch的准确率"></a>计算每个 batch 的准确率</h5><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">batch_accuracy</span>(<span class="params">pre, label</span>):</span><br><span class="line"> pre = pre.argmax(dim=<span class="number">1</span>)</span><br><span class="line"> correct = torch.eq(pre, label).<span class="built_in">sum</span>().<span class="built_in">float</span>().item()</span><br><span class="line"> accuracy = correct / <span class="built_in">float</span>(<span class="built_in">len</span>(label))</span><br><span class="line"></span><br><span class="line"> <span class="keyword">return</span> accuracy</span><br></pre></td></tr></tbody></table></figure><h5 id="训练"><a href="#训练" class="headerlink" title="训练"></a>训练</h5><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> epoch <span class="keyword">in</span> <span class="built_in">range</span>(EPOCHS):</span><br><span class="line"> step = -<span class="number">1</span></span><br><span class="line"> <span class="keyword">for</span> text, label <span class="keyword">in</span> train_loader:</span><br><span class="line"> <span class="comment"># tuple转list</span></span><br><span class="line"> text = <span class="built_in">list</span>(text)</span><br><span class="line"> label = <span class="built_in">list</span>(label)</span><br><span class="line"> label = <span class="built_in">list</span>(<span class="built_in">map</span>(<span class="built_in">int</span>, label))</span><br><span class="line"> <span class="comment"># 使用中文bert，生成句向量</span></span><br><span class="line"> sen_vec = bertclient.encode(text)</span><br><span class="line"> sen_vec = torch.tensor(sen_vec)</span><br><span class="line"> label = torch.LongTensor(label)</span><br><span class="line"> label = label.cuda()</span><br><span class="line"> <span class="comment"># 输入到网络中，反向传播</span></span><br><span class="line"> pre = net(sen_vec).cuda()</span><br><span class="line"> loss = criterion(pre, label)</span><br><span class="line"> optimizer.zero_grad()</span><br><span class="line"> loss.backward()</span><br><span class="line"> optimizer.step()</span><br><span class="line"> <span class="comment"># 更新loss曲线，并计算准确率</span></span><br><span class="line"> step = step + <span class="number">1</span></span><br><span class="line"> flag = flag + <span class="number">1</span></span><br><span class="line"> <span class="keyword">if</span> step % <span class="number">100</span> == <span class="number">0</span>:</span><br><span class="line"> acc = batch_accuracy(pre, label)</span><br><span class="line"> <span class="built_in">print</span>(<span class="string">'epoch:{} | batch:{} | acc:{} | loss:{}'</span>.<span class="built_in">format</span>(epoch, step, acc, loss.item()))</span><br></pre></td></tr></tbody></table></figure><h5 id="测试"><a href="#测试" class="headerlink" title="测试"></a>测试</h5><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">net.load_state_dict(torch.load(<span class="string">'net.pt'</span>))</span><br><span class="line"></span><br><span class="line">test_result = []</span><br><span class="line"><span class="keyword">for</span> item <span class="keyword">in</span> test_dataset:</span><br><span class="line"></span><br><span class="line"> sen_vec = bertclient.encode([item[<span class="number">1</span>]])</span><br><span class="line"> sen_vec = torch.tensor(sen_vec)</span><br><span class="line"></span><br><span class="line"> <span class="keyword">with</span> torch.no_grad():</span><br><span class="line"> pre = net(sen_vec).cuda()</span><br><span class="line"> pre = pre.argmax(dim=<span class="number">1</span>)</span><br><span class="line"> pre = pre.item()</span><br><span class="line"> test_result.append([item[<span class="number">0</span>], pre])</span><br><span class="line"></span><br><span class="line"> <span class="comment"># 写入csv文件</span></span><br><span class="line"> df = pd.DataFrame(test_result)</span><br><span class="line"> df.to_csv(<span class="string">'test_result.csv'</span>,index=<span class="literal">False</span>, header=[<span class="string">'id'</span>, <span class="string">'label'</span>])</span><br></pre></td></tr></tbody></table></figure><h5 id="训练可视化"><a href="#训练可视化" class="headerlink" title="训练可视化"></a>训练可视化</h5><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056403020/2020115154120.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056403020/2020115154120.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="训练可视化"></p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056431789/2020115154146.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056431789/2020115154146.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="训练可视化"></p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056458099/2020115154159.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056458099/2020115154159.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="训练可视化"></p><h5 id="控制台输出"><a href="#控制台输出" class="headerlink" title="控制台输出"></a>控制台输出</h5><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1652056483199/2020115154221.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1652056483199/2020115154221.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="控制台输出"></p></div><div class="story post-story"><h2 id="参考文献"><a href="#参考文献" class="headerlink" title="参考文献"></a>参考文献</h2><p><a target="_blank" rel="external nofollow noopener noreferrer" href="https://zhuanlan.zhihu.com/p/49271699">从 Word Embedding 到 Bert 模型 — 自然语言处理中的预训练技术发展史</a></p><p><a target="_blank" rel="external nofollow noopener noreferrer" href="https://www.jianshu.com/p/e7d8caa13b21">Transformer</a></p><p><a target="_blank" rel="external nofollow noopener noreferrer" href="https://zhuanlan.zhihu.com/p/37601161">深度学习中的注意力模型</a></p><p><a target="_blank" rel="external nofollow noopener noreferrer" href="https://github.com/google-research/bert">谷歌的 bert</a></p><p><a target="_blank" rel="external nofollow noopener noreferrer" href="https://github.com/hanxiao/bert-as-service">肖涵博士的 bert-as-service</a></p><p><a target="_blank" rel="external nofollow noopener noreferrer" href="https://hanxiao.io/2019/01/02/Serving-Google-BERT-in-Production-using-Tensorflow-and-ZeroMQ/">Serving-Google-BERT-in-Production-using-Tensorflow-and-ZeroMQ</a></p><p><a target="_blank" rel="external nofollow noopener noreferrer" href="https://github.com/thunlp/CAIL">CAIL2018 数据集</a></p><p><a target="_blank" rel="external nofollow noopener noreferrer" href="https://github.com/thunlp/OpenCLaP">刑事文书 BERT:</a></p><p><a target="_blank" rel="external nofollow noopener noreferrer" href="https://zhuanlan.zhihu.com/p/115802478">tensorboard 使用详解</a></p><p><a target="_blank" rel="external nofollow noopener noreferrer" href="https://www.cnblogs.com/fanghao/p/10256287.html">Visdom 可视化工具</a></p><p><a target="_blank" rel="external nofollow noopener noreferrer" href="https://www.datafountain.cn/competitions/350">互联网新闻情感分析</a></p></div></div><div class="footer"><div class="copyright license"><div class="license-title">BERT 预训练模型及其应用案例</div><div class="license-link"><a href="https://blog.mhuig.top/p/48f700eb/">https://blog.mhuig.top/p/48f700eb/</a></div><div class="license-meta"><div class="license-meta-item"><div class="license-meta-title">本文作者</div><div class="license-meta-text">MHuiG</div></div><div class="license-meta-item"><div class="license-meta-title">发布于</div><div class="license-meta-text">2020年11月5日</div></div><div class="license-meta-item"><div class="license-meta-title">许可协议</div><div class="license-meta-text"><a href="https://creativecommons.org/licenses/by-nc-sa/4.0/deed.zh#" target="_blank" rel="external nofollow noopener noreferrer">CC BY-NC-SA 4.0</a></div></div></div><div class="license-statement">署名-非商业性使用-相同方式共享 4.0 国际。</div></div></div><div class="article-meta" id="bottom"><div class="new-meta-box"><div class="new-meta-item category"><i class="fa-duotone fa-folder-open fa-fw" aria-hidden="true"></i> <a class="category-link" href="/categories/NLP/">NLP</a> <span hidden itemprop="about" itemscope itemtype="http://schema.org/Thing"><a href="/categories/NLP/" itemprop="url"><span itemprop="name">NLP</span></a></span></div><div class="new-meta-item meta-tags"><a class="tag" href="/tags/NLP/" rel="nofollow"><i class="fa-duotone fa-hashtag fa-fw" aria-hidden="true"></i><p>NLP</p></a></div> <span hidden itemprop="keywords">NLP</span><div class="new-meta-item date" itemprop="dateCreated datePublished" datetime="2020-11-05T15:47:27+08:00"><a class="notlink"><i class="fa-duotone fa-calendar-alt fa-fw" aria-hidden="true"></i><p>发布于：2020年11月5日</p></a></div><div class="new-meta-item date" itemprop="dateModified" datetime="2020-11-05T15:47:27+08:00"><a class="notlink"><i class="fa-duotone fa-edit fa-fw" aria-hidden="true"></i><p>更新于：2020年11月5日</p></a></div></div></div><div class="prev-next"><a class="prev" href="/p/9feab2fa/"><p class="title"><i class="fa-solid fa-chevron-left" aria-hidden="true"></i>互联网进化！</p><p class="content">简介《互联网进化论》书中提出” 互联网的未来功能和结构将于人类大脑高度相似，也将具备互联网虚拟感觉，虚拟运动，虚拟中枢，虚拟运动神经系统”, 并绘制了一幅互联网虚拟大脑结构图. 云计算是互联...</p></a><a class="next" href="/p/56213be8/"><p class="title">尝试使用 GPU 加速计算<i class="fa-solid fa-chevron-right" aria-hidden="true"></i></p><p class="content">大规模训练，gpu 和 cpu 速度差别很大。 概述GPU CPU (Central Processing Unit) 即中央处理器。 GPU (Graphics Processing Un...</p></a></div><div class="recommended-article"><div class="recommended-article-header"><i class="fa-solid fa-bookmark fa-fw" aria-hidden="true"></i> <span>推荐阅读</span></div><div class="recommended-article-group"> <a class="recommended-article-item" href="/p/175a1706/" title="尝试在博客中添加简易文章推荐功能" rel="bookmark"><img src="https://static.mhuig.top/npm/mhgoos@0.0.1663317354137/tf-idf.png" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1663317354137/tf-idf.png" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="尝试在博客中添加简易文章推荐功能"> <span class="title">尝试在博客中添加简易文章推荐功能</span></a> <a class="recommended-article-item" href="/p/483290b5/" title="特征和分类器" rel="bookmark"><img src="https://static.mhuig.top/npm/imbox@0.0.15/c/4.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.15/c/4.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="特征和分类器"> <span class="title">特征和分类器</span></a> <a class="recommended-article-item" href="/notes/Hadoop/mapreduce.html" title="大数据处理技术-Hadoop-MapReduce" rel="bookmark"><img src="https://static.mhuig.top/npm/imbox@0.0.15/c/82.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.15/c/82.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="大数据处理技术-Hadoop-MapReduce"> <span class="title">大数据处理技术-Hadoop-MapReduce</span></a> <a class="recommended-article-item" href="/notes/Hadoop/hdoop-arch.html" title="大数据处理技术-hadoop的架构模型（1.x，2.x的各种架构模型介绍）" rel="bookmark"><img src="https://static.mhuig.top/npm/imbox@0.0.15/c/106.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.15/c/106.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="大数据处理技术-hadoop的架构模型（1.x，2.x的各种架构模型介绍）"> <span class="title">大数据处理技术-hadoop的架构模型（1.x，2.x的各种架构模型介绍）</span></a> <a class="recommended-article-item" href="/notes/Spark/als.html" title="基于Audioscrobbler数据集的音乐推荐(pyspark)" rel="bookmark"><img src="https://static.mhuig.top/npm/imbox@0.0.15/c/66.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.15/c/66.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="基于Audioscrobbler数据集的音乐推荐(pyspark)"> <span class="title">基于Audioscrobbler数据集的音乐推荐(pyspark)</span></a> <a class="recommended-article-item" href="/p/110850a/" title="手把手教你用 Python 开始第一个机器学习项目" rel="bookmark"><img src="https://static.mhuig.top/npm/imbox@0.0.15/c/7.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.15/c/7.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="手把手教你用 Python 开始第一个机器学习项目"> <span class="title">手把手教你用 Python 开始第一个机器学习项目</span></a></div></div></article><article class="post white-box shadow floatable blur" id="comments"><span hidden><meta itemprop="discussionUrl" content="/p/48f700eb/#comments"></span><p ct><i class="fa-duotone fa-comments"></i> 留言区</p><div id="layoutHelper-comments"></div></article></div><aside id="l_side" itemscope itemtype="http://schema.org/WPSideBar"><div class="widget-sticky pjax"><section class="widget toc-wrapper desktop mobile" id="toc-div"><header><i class="fa-duotone fa-list fa-fw" aria-hidden="true"></i> <span class="name">本文目录</span></header><div class="content"><ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%9B%BE%E5%83%8F%E9%A2%86%E5%9F%9F%E7%9A%84%E9%A2%84%E8%AE%AD%E7%BB%83"><span class="toc-text">图像领域的预训练</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Word-Embedding"><span class="toc-text">Word Embedding</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B"><span class="toc-text">语言模型</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B"><span class="toc-text">神经网络语言模型</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Word2Vec"><span class="toc-text">Word2Vec</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Word-Embedding%E7%9A%84%E4%BD%BF%E7%94%A8"><span class="toc-text">Word Embedding 的使用</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Word-Embedding%E7%9A%84%E9%97%AE%E9%A2%98"><span class="toc-text">Word Embedding 的问题</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BB%8EWord-Embedding%E5%88%B0ELMO"><span class="toc-text">从 Word Embedding 到 ELMO</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BB%8EWord-Embedding%E5%88%B0GPT"><span class="toc-text">从 Word Embedding 到 GPT</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#Transformer"><span class="toc-text">Transformer</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%87%AA%E6%B3%A8%E6%84%8F%E5%8A%9B%E6%9C%BA%E5%88%B6%E6%A8%A1%E5%9E%8B"><span class="toc-text">自注意力机制模型</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#GPT%E5%A6%82%E4%BD%95%E4%BD%BF%E7%94%A8"><span class="toc-text">GPT 如何使用</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BB%8EGPT%E5%92%8CELMO%E5%8F%8Aword2Vec%E5%88%B0Bert"><span class="toc-text">从 GPT 和 ELMO 及 word2Vec 到 Bert</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#NLP%E7%9A%84%E5%9B%9B%E5%A4%A7%E4%BB%BB%E5%8A%A1"><span class="toc-text">NLP 的四大任务</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#BERT%E7%9A%84%E5%BA%94%E7%94%A8%E6%A1%88%E4%BE%8B"><span class="toc-text">BERT 的应用案例</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%B8%8B%E8%BD%BDbert%E9%A2%84%E8%AE%AD%E7%BB%83%E6%A8%A1%E5%9E%8B"><span class="toc-text">下载 bert 预训练模型</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%AE%89%E8%A3%85bert-as-service"><span class="toc-text">安装 bert-as-service</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%A1%88%E4%BE%8B%E4%B8%80-%E6%9F%A5%E6%89%BE%E6%9C%80%E7%9B%B8%E8%BF%91%E7%9A%84%E5%8F%A5%E5%AD%90"><span class="toc-text">案例一 查找最相近的句子</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%A1%88%E4%BE%8B%E4%BA%8C-%E7%AE%80%E5%8D%95%E6%A8%A1%E7%B3%8A%E6%90%9C%E7%B4%A2"><span class="toc-text">案例二 简单模糊搜索</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%A1%88%E4%BE%8B%E4%B8%89-%E6%B3%95%E6%9D%A1%E6%8E%A8%E8%8D%90"><span class="toc-text">案例三 法条推荐</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E4%BB%8B%E7%BB%8D"><span class="toc-text">介绍</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E8%AF%B4%E6%98%8E"><span class="toc-text">数据说明</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%AE%9E%E7%8E%B0%E6%B5%81%E7%A8%8B"><span class="toc-text">实现流程</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#%E5%88%9B%E5%BB%BA%E4%B8%80%E4%B8%AA%E8%BF%9E%E6%8E%A5%E5%88%B0BertServer%E7%9A%84BertClient"><span class="toc-text">创建一个连接到 BertServer 的 BertClient</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E8%8E%B7%E5%8F%96%E7%BC%96%E7%A0%81%E5%90%91%E9%87%8F%E5%92%8C%E6%A0%87%E7%AD%BE"><span class="toc-text">获取编码向量和标签</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E6%9E%84%E5%BB%BATensorFlow-DNN-%E6%A8%A1%E5%9E%8B%E7%9A%84%E5%88%86%E7%B1%BB%E5%99%A8"><span class="toc-text">构建 TensorFlow DNN 模型的分类器</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E8%AE%AD%E7%BB%83%E5%92%8C%E8%AF%84%E4%BC%B0"><span class="toc-text">训练和评估</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E8%BF%90%E8%A1%8Ctensorboard%E5%8F%AF%E8%A7%86%E5%8C%96%E8%AE%AD%E7%BB%83%E8%BF%87%E7%A8%8B"><span class="toc-text">运行 tensorboard 可视化训练过程</span></a></li></ol></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%A1%88%E4%BE%8B%E5%9B%9B-%E4%BA%92%E8%81%94%E7%BD%91%E6%96%B0%E9%97%BB%E6%83%85%E6%84%9F%E5%88%86%E6%9E%90"><span class="toc-text">案例四 互联网新闻情感分析</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E4%BB%8B%E7%BB%8D-1"><span class="toc-text">介绍</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E8%AF%B4%E6%98%8E-1"><span class="toc-text">数据说明</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%AE%9E%E7%8E%B0%E6%B5%81%E7%A8%8B-1"><span class="toc-text">实现流程</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#%E5%8A%A0%E8%BD%BD%E6%95%B0%E6%8D%AE%E9%9B%86"><span class="toc-text">加载数据集</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E7%BD%91%E7%BB%9C%E7%B1%BB%EF%BC%8C%E5%85%A8%E8%BF%9E%E6%8E%A5%E5%B1%82"><span class="toc-text">网络类，全连接层</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E8%AE%A1%E7%AE%97%E6%AF%8F%E4%B8%AAbatch%E7%9A%84%E5%87%86%E7%A1%AE%E7%8E%87"><span class="toc-text">计算每个 batch 的准确率</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E8%AE%AD%E7%BB%83"><span class="toc-text">训练</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E6%B5%8B%E8%AF%95"><span class="toc-text">测试</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E8%AE%AD%E7%BB%83%E5%8F%AF%E8%A7%86%E5%8C%96"><span class="toc-text">训练可视化</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E6%8E%A7%E5%88%B6%E5%8F%B0%E8%BE%93%E5%87%BA"><span class="toc-text">控制台输出</span></a></li></ol></li></ol></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%8F%82%E8%80%83%E6%96%87%E7%8C%AE"><span class="toc-text">参考文献</span></a></li></ol></div></section></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div></aside><pjax><script>window.pdata={},pdata.ispage=!0,pdata.commentPath="",pdata.commentPlaceholder="",pdata.commentConfig={};var l_header=document.getElementById("l_header");l_header.classList.add("show");var cover_wrapper=document.querySelector("#l_cover .cover-wrapper"),scroll_down=document.getElementById("scroll-down");cover_wrapper.id="none",cover_wrapper.style.display="none",scroll_down.style.display="none"</script></pjax></div><footer class="footer clearfix" itemscope itemtype="http://schema.org/WPFooter"><br><br><div class="aplayer-container"><div class="aplayer-local"></div></div><br><div class="social-wrapper" itemprop="about" itemscope itemtype="http://schema.org/Thing"><a href="/atom.xml" class="social fa-duotone fa-rss flat-btn" target="_blank" rel="external nofollow noopener noreferrer" itemprop="url"></a><a href="https://mhuig.top/contact/" class="social fa-duotone fa-envelope flat-btn" target="_blank" rel="external nofollow noopener noreferrer" itemprop="url"></a><a href="https://github.com/MHuiG" class="social fa-brands fa-github flat-btn" target="_blank" rel="external nofollow noopener noreferrer" itemprop="url"></a><a href="https://t.me/MHuiG" class="social fa-brands fa-telegram flat-btn" target="_blank" rel="external nofollow noopener noreferrer" itemprop="url"></a></div><div><p>博客内容遵循 <a target="_blank" rel="external nofollow noopener noreferrer" href="https://creativecommons.org/licenses/by-nc-sa/4.0/deed.zh">署名-非商业性使用-相同方式共享 4.0 国际 (CC BY-NC-SA 4.0) 协议</a></p></div><div class="copyright"><p><a href="https://icp.gov.moe" target="_blank" rel="external nofollow noopener noreferrer">萌ICP备</a> <a href="https://icp.gov.moe/?keyword=2020012138" target="_blank" rel="external nofollow noopener noreferrer">2020012138号</a><br><a target="_blank" rel="noopener" href="https://mhuig.top/">Copyright © Since 2018 MHuiG</a></p></div></footer><a id="s-top" class="fa-solid fa-arrow-up fa-fw" title="top"></a></div></div><div><script>volantis.dom.bodyAnchor=volantis.dom.$(document.getElementById("safearea")),volantis.dom.topBtn=volantis.dom.$(document.getElementById("s-top")),volantis.dom.wrapper=volantis.dom.$(document.getElementById("wrapper")),volantis.dom.coverAnchor=volantis.dom.$(document.querySelector("#l_cover .cover-wrapper")),volantis.dom.switcher=volantis.dom.$(document.querySelector("#l_header .switcher .s-search")),volantis.dom.header=volantis.dom.$(document.getElementById("l_header")),volantis.dom.search=volantis.dom.$(document.querySelector("#l_header .m_search")),volantis.dom.mPhoneList=volantis.dom.$(document.querySelectorAll("#l_header .m-phone .list-v"))</script><script>volantis.css("https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/libs/@fortawesome/fontawesome-pro/css/all.min.css")</script><script src="/js/app.js?v=196690b6a0"></script><script>const rootElement=document.documentElement,darkModeStorageKey="color-scheme",rootElementDarkModeAttributeName="color-scheme",setLS=(e,t)=>{localStorage.setItem(e,t)},removeLS=e=>{localStorage.removeItem(e)},getLS=e=>localStorage.getItem(e),getModeFromCSSMediaQuery=()=>window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light",resetRootDarkModeAttributeAndLS=()=>{rootElement.removeAttribute("color-scheme"),removeLS("color-scheme")},validColorModeKeys={dark:!0,light:!0},applyCustomDarkModeSettings=e=>{const t=e||getLS("color-scheme");getCustomDarkMode(),t===getModeFromCSSMediaQuery()?resetRootDarkModeAttributeAndLS():validColorModeKeys[t]?rootElement.setAttribute("color-scheme",t):resetRootDarkModeAttributeAndLS()},invertDarkModeObj={dark:"light",light:"dark"},getCustomDarkMode=()=>{let e=getLS("color-scheme");if(validColorModeKeys[e])e=invertDarkModeObj[e];else{if(null!==e)return;e=invertDarkModeObj[getModeFromCSSMediaQuery()]}volantis.dark.mode="dark"==e?"light":"dark"},toggleCustomDarkMode=()=>{let e=getLS("color-scheme");if(validColorModeKeys[e])e=invertDarkModeObj[e];else{if(null!==e)return;e=invertDarkModeObj[getModeFromCSSMediaQuery()]}return setLS("color-scheme",e),e};function bindToggleButton(){document.querySelectorAll("#wrapper .toggle-mode-btn,#rightmenu-wrapper .toggle-mode-btn").forEach((function(e){volantis.dom.$(e).on("click",volantis.dark.toggle)}))}volantis.dark.toggle=()=>{const e=toggleCustomDarkMode();applyCustomDarkModeSettings(e),volantis.dark.method.toggle.start()},applyCustomDarkModeSettings(),document.addEventListener("DOMContentLoaded",(()=>{volantis.requestAnimationFrame(bindToggleButton)})),volantis.pjax.push(bindToggleButton);const darkModelListeners={dark:e=>{e.matches&&(volantis.dark.mode="dark"),volantis.dark.method.toggle.start()},light:e=>{e.matches&&(volantis.dark.mode="light"),volantis.dark.method.toggle.start()}};window.matchMedia("(prefers-color-scheme: dark)").addListener(darkModelListeners.dark),window.matchMedia("(prefers-color-scheme: light)").addListener(darkModelListeners.light)</script><script>function loadIssuesJS(){null!=document.getElementById("sites-api")&&"undefined"==typeof SitesJS&&volantis.js("/js/plugins/tags/sites.js?v=a7e55dacdf");null!=document.getElementById("friends-api")&&"undefined"==typeof FriendsJS&&volantis.js("/js/plugins/tags/friends.js?v=187f37f894");null!=document.getElementById("contributors-api")&&"undefined"==typeof ContributorsJS&&volantis.js("/js/plugins/tags/contributors.js?v=16e248333a")}loadIssuesJS(),volantis.pjax.push((()=>{loadIssuesJS()}))</script><script defer="defer" src="https://cdn.bootcdn.net/ajax/libs/vanilla-lazyload/17.1.0/lazyload.min.js"></script><script>window.lazyLoadOptions={elements_selector:".lazyload",threshold:0},window.addEventListener("LazyLoad::Initialized",(function(n){window.lazyLoadInstance=n.detail.instance}),!1),document.addEventListener("DOMContentLoaded",(function(){lazyLoadInstance.update()})),document.addEventListener("pjax:complete",(function(){lazyLoadInstance.update()}))</script><script>window.FPConfig={delay:0,ignoreKeywords:["#"],maxRPS:6,hoverDelay:0}</script><script defer="defer" src="https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/libs/flying-pages/flying-pages.min.js"></script><script type="text/javascript">function pjax_scrollrebeal(){ScrollReveal().reveal("#l_main .reveal",{distance:"32px",duration:"800",interval:"20",scale:"1",easing:"ease-out"})}function init_scrollrebeal(){"undefined"==typeof ScrollReveal?volantis.requestAnimationFrame(init_scrollrebeal):pjax_scrollrebeal()}volantis.js("https://cdn.bootcdn.net/ajax/libs/scrollReveal.js/4.0.9/scrollreveal.min.js"),document.addEventListener("DOMContentLoaded",init_scrollrebeal),volantis.pjax.push(pjax_scrollrebeal,"pjax_scrollrebeal",setRequestAnimationFrame=!1)</script><script>volantis.css("https://cdn.bootcdn.net/ajax/libs/aplayer/1.10.1/APlayer.min.css"),volantis.js("https://cdn.bootcdn.net/ajax/libs/aplayer/1.10.1/APlayer.min.js").then((()=>{document.querySelectorAll(".aplayer-local").forEach((e=>{new APlayer({container:e,fixed:!1,order:"list",volume:"0.7",autoplay:!1,loop:"all",theme:"#1BCDFC",listMaxHeight:"320px",listFolded:"true",preload:"auto",lrcType:3,audio:[{name:"虹之间",artist:"钢琴版伴奏",url:"https://blog.mhuig.top/music-Archive/BetweenTheRainbow.mp3",lrc:"https://blog.mhuig.top/music-Archive/BetweenTheRainbow.lrc",cover:"https://blog.mhuig.top/music-Archive/BetweenTheRainbow.jpg"},{name:"Avem",artist:"Alan Walker",url:"https://blog.mhuig.top/music-Archive/Avem.mp3",lrc:"https://blog.mhuig.top/music-Archive/Avem.lrc",cover:"https://blog.mhuig.top/music-Archive/Avem.jpg"},{name:"Fly",artist:"Marshmello / Leau Culver",url:"https://blog.mhuig.top/music-Archive/fly.mp3",lrc:"https://blog.mhuig.top/music-Archive/fly.lrc",cover:"https://blog.mhuig.top/music-Archive/fly.jpg"},{name:"Alone",artist:"Marshmello",url:"https://blog.mhuig.top/music-Archive/alone.mp3",lrc:"https://blog.mhuig.top/music-Archive/alone.lrc",cover:"https://blog.mhuig.top/music-Archive/alone.jpg"}]})}))}))</script><script>function check_giscus(){return"dark"===volantis.dark.mode?volantis.giscus.Theme="dark":volantis.giscus.Theme="light",document.getElementById("giscus_container")}function pjax_giscus(){const t=check_giscus();if(!t)return;let s=Object.assign({theme:{light:"light",dark:"dark"},repo:"MHuiG/talk","repo-id":"MDEwOlJlcG9zaXRvcnkyODQ4NDg1NTg=",category:"Giscus","category-id":"DIC_kwDOEPpxrs4B_X67",mapping:"pathname","reactions-enabled":"1","emit-metadata":"1",lang:"zh-CN","input-position":"top"},pdata.commentConfig);const e=document.createElement("script");e.setAttribute("src","https://giscus.app/client.js"),Object.keys(s).forEach((t=>{"theme"!=t&&e.setAttribute("data-"+t,s[t])})),e.setAttribute("data-theme",volantis.giscus.Theme),e.setAttribute("crossorigin","anonymous"),t.appendChild(e)}function dark_giscus(){if(!check_giscus())return;const t={setConfig:{theme:volantis.giscus.Theme}};document.querySelector("iframe.giscus-frame").contentWindow.postMessage({giscus:t},"https://giscus.app")}volantis.layoutHelper("comments",'<div id="giscus_container"></div>'),volantis.giscus={},pjax_giscus(),volantis.pjax.push(pjax_giscus),volantis.dark.push(dark_giscus)</script><script>async function loadSearchScript(){return volantis.js("/js/search/hexo.js?v=e2a1e6a8d9");}function loadSearchService(){loadSearchScript(),document.querySelectorAll(".input.u-search-input").forEach((e=>{e.removeEventListener("focus",loadSearchService,!1)})),document.querySelectorAll(".u-search-form").forEach((e=>{e.addEventListener("submit",(e=>{e.preventDefault()}),!1)}))}function OpenSearch(e){"undefined"==typeof SearchService?loadSearchScript().then((()=>{SearchService.setQueryText(e),SearchService.search()})):(SearchService.setQueryText(e),SearchService.search())}if(window.location.search&&/^\?s=/g.test(window.location.search)){OpenSearch(decodeURI(window.location.search).replace(/\ /g,"-").replace(/^\?s=/g,""))}document.querySelectorAll(".input.u-search-input").forEach((e=>{e.addEventListener("focus",loadSearchService,!1)}))</script><script>function pjax_highlightjs_copyCode(){(document.querySelector(".highlight .code pre")||document.querySelector(".article pre code"))&&VolantisApp.utilCopyCode(".highlight .code pre, .article pre code")}volantis.requestAnimationFrame(pjax_highlightjs_copyCode),volantis.pjax.push(pjax_highlightjs_copyCode)</script><script>function load_swiper(){document.querySelectorAll(".swiper-container")[0]&&(volantis.css("https://cdn.bootcdn.net/ajax/libs/Swiper/8.3.2/swiper-bundle.min.css"),volantis.js("https://cdn.bootcdn.net/ajax/libs/Swiper/8.3.2/swiper-bundle.min.js").then((()=>{pjax_swiper()})))}function pjax_swiper(){volantis.swiper=new Swiper(".swiper-container",{slidesPerView:"auto",spaceBetween:8,centeredSlides:!0,loop:!0,pagination:{el:".swiper-pagination",clickable:!0},navigation:{nextEl:".swiper-button-next",prevEl:".swiper-button-prev"}})}load_swiper(),volantis.pjax.push((()=>{document.querySelectorAll(".swiper-container")[0]&&(void 0===volantis.swiper?load_swiper():pjax_swiper())}))</script><script>volantis.css("https://cdn.bootcdn.net/ajax/libs/pace/1.2.4/themes/blue/pace-theme-minimal.min.css"),volantis.js("https://cdn.bootcdn.net/ajax/libs/pace/1.2.4/pace.min.js").then((()=>{Pace.options.restartOnPushState=!1,volantis.pjax.send(Pace.restart),volantis.scroll.push((()=>{Pace.stop(),Pace.trigger("start"),Pace.bar.update(100*volantis.scroll.progress()),document.body.className=document.body.className.replaceAll("pace-running ","")}),"阅读进度")}))</script><pjax></pjax><script>function listenSidebarTOC(){const t=document.querySelectorAll(".toc li");if(!t.length)return;const e=[];Array.from(t).forEach((t=>{const n=t.querySelector(".toc-link"),i=document.getElementById(n.getAttribute("href")?decodeURI(n.getAttribute("href")).replace("#",""):n.getAttribute("toc-action").split("toc-")[1]);return e.push(i),n.getAttribute("href")&&(n.setAttribute("toc-action","toc-"+decodeURI(n.getAttribute("href")).replace("#","")),n.removeAttribute("href")),i&&i.id&&n.addEventListener("click",(t=>{t.preventDefault(),volantis.scroll.to(i,{addTop:5,observer:!0}),history.pushState(null,document.title,"#"+i.id)})),i}));function n(t){if(t.classList.contains("active-current"))return;document.querySelectorAll(".toc .active").forEach((t=>{t.classList.remove("active","active-current")})),t.classList.add("active","active-current");let e=t.parentNode;for(;!e.matches(".toc");)e.matches("li")&&e.classList.add("active"),e=e.parentNode}volantis.activateNavIndex=0,n(t[volantis.activateNavIndex]),e[0]&&volantis.scroll.push((()=>{if(e[0].getBoundingClientRect().top>=0)volantis.activateNavIndex=0;else if(e[e.length-1].getBoundingClientRect().top<0)volantis.activateNavIndex=e.length-1;else for(let t=0;t<e.length;t++){const n=e[t],i=e[(t+1)%e.length];if(n.getBoundingClientRect().top<0&&i.getBoundingClientRect().top>=0){volantis.activateNavIndex=t;break}}n(t[volantis.activateNavIndex])}))}document.addEventListener("DOMContentLoaded",(()=>{volantis.requestAnimationFrame(listenSidebarTOC)})),volantis.pjax.push(listenSidebarTOC)</script><script>try{let e=(e,t,n=2)=>Math.abs(e-t)<=n,t=(t,n)=>!e(t.width,n.width)||!e(t.height,n.height),n=new WeakMap,i=(e,i=e.getClientBoundingRect())=>{let o=n.get(e);o&&!t(o,i)||(n.set(e,i),e.style["contain-intrinsic-size"]=`${i.width}px ${i.height}px`)},o=new IntersectionObserver(((e,t)=>{e.forEach((e=>{i(e.target,e.boundingClientRect)}))}),{rootMargin:"500px 0px 500px 0px"}),r=new ResizeObserver(((e,t)=>{e.forEach((e=>{i(e.target,e.contentRect)}))})),s=e=>{let t=document.querySelectorAll(e);t.length&&(t.forEach((e=>{o.observe(e),r.observe(e)})),requestAnimationFrame((()=>{requestAnimationFrame((()=>{t[0].style["content-visibility"]="auto"}))})))},a=()=>{"content-visibility"in document.documentElement.style&&s(".post-story")};a(),volantis.pjax.push(a)}catch(e){console.log(e)}</script><script>document.onreadystatechange=function(){if("complete"==document.readyState){const{saveData:e,effectiveType:t}=navigator.connection||navigator.mozConnection||navigator.webkitConnection||{};("none"==getComputedStyle(document.querySelector("#safearea"),null).display||e||/2g/.test(t))&&(document.querySelectorAll(".reveal").forEach((function(e){e.style.opacity="1"})),document.querySelector("#safearea").style.display="block")}}</script><script type="application/ld+json">[{"@context":"http://schema.org","@type":"Organization","name":"MHuiG's Blog","url":"https://blog.mhuig.top/","logo":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3","width":192,"height":192}},{"@context":"http://schema.org","@type":"Person","name":"MHuiG","image":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3"},"url":"https://blog.mhuig.top/","sameAs":["https://github.com/MHuiG","https://twitter.com/iMHuiG","https://t.me/MHuiG","https://keybase.io/MHuiG"],"description":"「Be Yourself, Make a Difference.」"},{"@context":"http://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"item":{"@id":"https://blog.mhuig.top/","name":"MHuiG"}},{"@type":"ListItem","position":2,"item":{"@id":"https://blog.mhuig.top/categories/NLP/","name":"NLP"}},{"@type":"ListItem","position":3,"item":{"@id":"https://blog.mhuig.top/p/48f700eb/","name":"BERT 预训练模型及其应用案例"}}]},{"@context":"http://schema.org","@type":"WebSite","name":"MHuiG's Blog","url":"https://blog.mhuig.top/","keywords":"Blog, IT, BigData, MHuiG, @MHuiG, iMHuiG, 博客, 梦幻岛, Neverland, 技术博客","description":"MHuiG&#39;s Blog (MHuiG的博客) MHuiG&#39;s Neverland（MHuiG的梦幻岛） —— MHuiG(@MHuiG) 随便写写画画的地方 - 技术博客","author":{"@type":"Person","name":"MHuiG","image":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3"},"url":"https://blog.mhuig.top/","description":"「Be Yourself, Make a Difference.」"},"publisher":{"@type":"Organization","name":"MHuiG's Blog","url":"https://blog.mhuig.top/","logo":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3","width":192,"height":192}},"potentialAction":{"@type":"SearchAction","name":"Coco | The Cat of MHuiG","target":{"@type":"EntryPoint","urlTemplate":"https://blog.mhuig.top/havefun/Coco/?s={search_term_string}"},"query-input":"required name=search_term_string"}},{"@context":"http://schema.org","@type":"BlogPosting","headline":"BERT 预训练模型及其应用案例","description":"预训练模型最开始是在图像领域提出的，获得了良好的效果，近几年才被广泛应用到自然语言处理各项任务中。\n\n(1) 2003 年 Bengio 提出神经网络语言模型 NNLM，从此统一了 NLP 的特征形式 ——Embedding；\n\n(2) 2013 年 Mikolov 提出词向量 Word2vec，延续 NNLM 又引入了大规模预训练（Pretrain）的思路；\n\n(3) 2017 年 Vaswani 提出 Transformer 模型，实现用一个模型处理多种 NLP 任务。\n\n(4) 基于 Transformer 架构，2018 年底开始出现一大批预训练语言模型 (3 个预训练代表性模型 BERT [2018]、XLNet [2019] 和 MPNet [2020])，刷新众多 NLP 任务，形成新的里程碑事件。\n\n","inLanguage":"zh-CN","mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.mhuig.top/p/48f700eb/"},"author":{"@type":"Person","name":"MHuiG","image":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3"},"url":"https://blog.mhuig.top/"},"publisher":{"@type":"Organization","name":"MHuiG's Blog","logo":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3","width":192,"height":192}},"url":"https://blog.mhuig.top/p/48f700eb/","wordCount":363,"datePublished":"2020-11-05T07:47:27.000Z","dateModified":"2020-11-05T07:47:27.000Z","articleSection":"NLP","keywords":"NLP","image":{"@type":"ImageObject","url":"https://static.mhuig.top/npm/mhgoos@0.0.1651974150318/2020116102430.webp","width":1024,"height":768}}]</script><script src="https://cdn.bootcdn.net/ajax/libs/pjax/0.2.8/pjax.min.js"></script><script>var pjax;document.addEventListener("DOMContentLoaded",(function(){pjax=new Pjax({elements:'a[href]:not([href^="#"]):not([href="javascript:void(0)"]):not([pjax-fancybox]):not([onclick="return false;"]):not([onclick="return!1"]):not([target="_blank"]):not([target="view_window"]):not([href$=".xml"])',selectors:["head title","head meta[name=keywords]","head meta[name=description]","#l_main","#pjax-header-nav-list",".pjax","pjax"],cacheBust:!1,timeout:5e3})})),document.addEventListener("pjax:send",(function(e){try{var t=window.location.pathname,n=e.triggerElement.href,a=[""];""!=a[0]&&a.forEach((e=>{-1==t.indexOf(e)&&-1==n.indexOf(e)||(window.location.href=n)}))}catch(e){}volantis.pjax.method.send.start()})),document.addEventListener("pjax:complete",(function(){document.querySelectorAll("script[data-pjax], .pjax-reload script").forEach((e=>{const t=e.text||e.textContent||e.innerHTML||"",n=document.createElement("script");Object.keys(e.attributes).forEach((t=>{n.setAttribute(e.attributes[t].nodeName,e.attributes[t].nodeValue)})),t&&n.appendChild(document.createTextNode(t)),e.parentNode.replaceChild(n,e)})),volantis.pjax.method.complete.start()})),document.addEventListener("pjax:error",(function(e){"pjax"===volantis.debug?(console.error(e),console.log("pjax error: \n"+JSON.stringify(e))):(volantis.pjax.method.error.start(),window.location.href=e.triggerElement.href)}))</script></div><pjax> <a style="display:none" target="_blank" rel="external nofollow noopener noreferrer" href="https://bot-trap.mhuig.top/">Are You A Robot?</a></pjax><script>top.location!=self.location&&(top.location=self.location),"mhuig.github.io"==window.location.host||"localhost:4000"==window.location.host||/mhuig\.top$/.test(window.location.host)||/kix\.moe$/.test(window.location.host)||/mhuig/.test(window.location.host)||/ipfs/.test(window.location.host)||(window.location.href="https://blog.mhuig.top/")</script></body></html>