Skip to content

L
linux
Commit 3104fb3d authored by Ingo Molnar's avatar Ingo Molnar
Browse files
Options

Merge branch 'for-mingo' of... 

Merge branch 'for-mingo' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu into core/rcu

Pull RCU changes from Paul E. McKenney:

 - Adding transitivity uniformly to rcu_node structure ->lock
   acquisitions.  (This is implemented by the first two commits
   on top of v4.4-rc2 due to the pervasive nature of this change.)

 - Documentation updates, including RCU requirements.

 - Expedited grace-period changes.

 - Miscellaneous fixes.

 - Linked-list fixes, courtesy of KTSAN.

 - Torture-test updates.

 - Late-breaking fix to sysrq-generated crash.
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parents ee9a7d2c 984cf355
Show whitespace changes
Inline Side-by-side
Showing with 7552 additions and 287 deletions
Documentation/RCU/Design/Requirements/GPpartitionReaders1.svg 0 → 100644
View file @ 3104fb3d
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
width="447.99197"
height="428.19299"
id="svg2"
version="1.1"
inkscape:version="0.48.3.1 r9886"
sodipodi:docname="GPpartitionReaders1.svg">
<defs
id="defs4">
<marker
inkscape:stockid="Arrow2Lend"
orient="auto"
refY="0"
refX="0"
id="Arrow2Lend"
style="overflow:visible">
<path
id="path3792"
style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
inkscape:connector-curvature="0" />
</marker>
<marker
inkscape:stockid="Arrow2Lstart"
orient="auto"
refY="0"
refX="0"
id="Arrow2Lstart"
style="overflow:visible">
<path
id="path3789"
style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
transform="matrix(1.1,0,0,1.1,1.1,0)"
inkscape:connector-curvature="0" />
</marker>
</defs>
<sodipodi:namedview
id="base"
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1.0"
inkscape:pageopacity="0.0"
inkscape:pageshadow="2"
inkscape:zoom="1.6184291"
inkscape:cx="223.99599"
inkscape:cy="214.0965"
inkscape:document-units="px"
inkscape:current-layer="layer1"
showgrid="false"
inkscape:window-width="979"
inkscape:window-height="836"
inkscape:window-x="571"
inkscape:window-y="335"
inkscape:window-maximized="0"
fit-margin-top="5"
fit-margin-left="5"
fit-margin-right="5"
fit-margin-bottom="5" />
<metadata
id="metadata7">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title></dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<g
inkscape:label="Layer 1"
inkscape:groupmode="layer"
id="layer1"
transform="translate(-28.441125,-185.60612)">
<flowRoot
xml:space="preserve"
id="flowRoot2985"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"><flowRegion
id="flowRegion2987"><rect
id="rect2989"
width="82.85714"
height="11.428572"
x="240"
y="492.36218" /></flowRegion><flowPara
id="flowPara2991"></flowPara></flowRoot> <g
id="g4433"
transform="translate(2,0)">
<text
sodipodi:linespacing="125%"
id="text2993"
y="-261.66608"
x="412.12299"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
xml:space="preserve"
transform="matrix(0,1,-1,0,0,0)"><tspan
y="-261.66608"
x="412.12299"
id="tspan2995"
sodipodi:role="line">synchronize_rcu()</tspan></text>
<g
id="g4417"
transform="matrix(0,1,-1,0,730.90257,222.4928)">
<path
style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-start:url(#Arrow2Lstart);marker-end:url(#Arrow2Lend)"
d="m 97.580736,477.4048 183.140664,0"
id="path2997"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
<path
style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
d="m 96.752718,465.38398 0,22.62742"
id="path4397"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
<path
style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
d="m 281.54942,465.38397 0,22.62742"
id="path4397-5"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
</g>
</g>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="112.04738"
y="268.18076"
id="text4429"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4431"
x="112.04738"
y="268.18076">WRITE_ONCE(a, 1);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="112.04738"
y="439.13766"
id="text4441"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4443"
x="112.04738"
y="439.13766">WRITE_ONCE(b, 1);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="255.60869"
y="309.29346"
id="text4445"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4447"
x="255.60869"
y="309.29346">r1 = READ_ONCE(a);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="255.14423"
y="520.61786"
id="text4449"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4451"
x="255.14423"
y="520.61786">WRITE_ONCE(c, 1);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="396.10254"
y="384.71124"
id="text4453"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4455"
x="396.10254"
y="384.71124">r2 = READ_ONCE(b);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="396.10254"
y="582.13617"
id="text4457"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4459"
x="396.10254"
y="582.13617">r3 = READ_ONCE(c);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="112.08231"
y="213.91006"
id="text4461"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4463"
x="112.08231"
y="213.91006">thread0()</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="252.34512"
y="213.91006"
id="text4461-6"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4463-0"
x="252.34512"
y="213.91006">thread1()</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="396.42557"
y="213.91006"
id="text4461-2"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4463-2"
x="396.42557"
y="213.91006">thread2()</tspan></text>
<rect
style="fill:none;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
id="rect4495"
width="436.28488"
height="416.4859"
x="34.648232"
y="191.10612" />
<path
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
d="m 183.14066,191.10612 0,417.193 -0.70711,0"
id="path4497"
inkscape:connector-curvature="0" />
<path
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
d="m 325.13867,191.10612 0,417.193 -0.70711,0"
id="path4497-5"
inkscape:connector-curvature="0" />
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="111.75929"
y="251.53981"
id="text4429-8"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4431-9"
x="111.75929"
y="251.53981">rcu_read_lock();</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="396.10254"
y="367.91556"
id="text4429-8-9"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4431-9-4"
x="396.10254"
y="367.91556">rcu_read_lock();</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="396.10254"
y="597.40289"
id="text4429-8-9-3"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4431-9-4-4"
x="396.10254"
y="597.40289">rcu_read_unlock();</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="111.75929"
y="453.15311"
id="text4429-8-9-3-1"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4431-9-4-4-6"
x="111.75929"
y="453.15311">rcu_read_unlock();</tspan></text>
<path
style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
d="m 33.941125,227.87568 436.284885,0 0,0.7071"
id="path4608"
inkscape:connector-curvature="0" />
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="394.94427"
y="345.66351"
id="text4648"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4650"
x="394.94427"
y="345.66351">QS</tspan></text>
<path
sodipodi:type="arc"
style="fill:none;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
id="path4652"
sodipodi:cx="358.85669"
sodipodi:cy="142.87541"
sodipodi:rx="10.960155"
sodipodi:ry="10.253048"
d="m 358.86939,132.62237 a 10.960155,10.253048 0 1 1 -0.0228,0"
transform="translate(36.441125,199.60612)"
sodipodi:start="4.7135481"
sodipodi:end="10.994651"
sodipodi:open="true" />
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="112.11968"
y="475.77856"
id="text4648-4"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4650-4"
x="112.11968"
y="475.77856">QS</tspan></text>
<path
sodipodi:type="arc"
style="fill:none;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
id="path4652-7"
sodipodi:cx="358.85669"
sodipodi:cy="142.87541"
sodipodi:rx="10.960155"
sodipodi:ry="10.253048"
d="m 358.86939,132.62237 a 10.960155,10.253048 0 1 1 -0.0228,0"
transform="translate(-246.38346,329.72117)"
sodipodi:start="4.7135481"
sodipodi:end="10.994651"
sodipodi:open="true" />
<path
sodipodi:type="arc"
style="fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
id="path4652-7-7"
sodipodi:cx="358.85669"
sodipodi:cy="142.87541"
sodipodi:rx="10.960155"
sodipodi:ry="10.253048"
d="m 358.86939,132.62237 a 10.960155,10.253048 0 1 1 -0.0228,0"
transform="translate(-103.65246,202.90878)"
sodipodi:start="4.7135481"
sodipodi:end="10.994651"
sodipodi:open="true" />
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="254.85066"
y="348.96619"
id="text4648-4-3"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4650-4-5"
x="254.85066"
y="348.96619">QS</tspan></text>
</g>
</svg>
Documentation/RCU/Design/Requirements/RCUApplicability.svg 0 → 100644
View file @ 3104fb3d
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Creator: fig2dev Version 3.2 Patchlevel 5d -->
<!-- CreationDate: Tue Mar 4 18:34:25 2014 -->
<!-- Magnification: 3.000 -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
width="1089.1382"
height="668.21368"
viewBox="-2121 -36 14554.634 8876.4061"
id="svg2"
version="1.1"
inkscape:version="0.48.3.1 r9886"
sodipodi:docname="RCUApplicability.svg">
<metadata
id="metadata40">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title />
</cc:Work>
</rdf:RDF>
</metadata>
<defs
id="defs38" />
<sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="849"
inkscape:window-height="639"
id="namedview36"
showgrid="false"
inkscape:zoom="0.51326165"
inkscape:cx="544.56912"
inkscape:cy="334.10686"
inkscape:window-x="149"
inkscape:window-y="448"
inkscape:window-maximized="0"
inkscape:current-layer="g4"
fit-margin-top="5"
fit-margin-left="5"
fit-margin-right="5"
fit-margin-bottom="5" />
<g
style="fill:none;stroke-width:0.025in"
id="g4"
transform="translate(-2043.6828,14.791398)">
<!-- Line: box -->
<rect
x="0"
y="0"
width="14400"
height="8775"
rx="0"
style="fill:#ffa1a1;stroke:#000000;stroke-width:21;stroke-linecap:butt;stroke-linejoin:miter"
id="rect6" />
<!-- Line: box -->
<rect
x="1350"
y="0"
width="11700"
height="6075"
rx="0"
style="fill:#ffff00;stroke:#000000;stroke-width:21;stroke-linecap:butt;stroke-linejoin:miter"
id="rect8" />
<!-- Line: box -->
<rect
x="2700"
y="0"
width="9000"
height="4275"
rx="0"
style="fill:#00ff00;stroke:#000000;stroke-width:21;stroke-linecap:butt;stroke-linejoin:miter"
id="rect10" />
<!-- Line: box -->
<rect
x="4050"
y="0"
width="6300"
height="2475"
rx="0"
style="fill:#87cfff;stroke:#000000;stroke-width:21;stroke-linecap:butt;stroke-linejoin:miter"
id="rect12" />
<!-- Text -->
<text
xml:space="preserve"
x="7200"
y="900"
font-style="normal"
font-weight="normal"
font-size="324"
id="text14"
sodipodi:linespacing="125%"
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"><tspan
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"
id="tspan3017">Read-Mostly, Stale &amp;</tspan></text>
<!-- Text -->
<text
xml:space="preserve"
x="7200"
y="1350"
font-style="normal"
font-weight="normal"
font-size="324"
id="text16"
sodipodi:linespacing="125%"
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"><tspan
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"
id="tspan3019">Inconsistent Data OK</tspan></text>
<!-- Text -->
<text
xml:space="preserve"
x="7200"
y="1800"
font-style="normal"
font-weight="normal"
font-size="324"
id="text18"
sodipodi:linespacing="125%"
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"><tspan
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"
id="tspan3021">(RCU Works Great!!!)</tspan></text>
<!-- Text -->
<text
xml:space="preserve"
x="7200"
y="3825"
font-style="normal"
font-weight="normal"
font-size="324"
id="text20"
sodipodi:linespacing="125%"
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"><tspan
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"
id="tspan3023">(RCU Works Well)</tspan></text>
<!-- Text -->
<text
xml:space="preserve"
x="7200"
y="3375"
font-style="normal"
font-weight="normal"
font-size="324"
id="text22"
sodipodi:linespacing="125%"
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"><tspan
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"
id="tspan3025">Read-Mostly, Need Consistent Data</tspan></text>
<!-- Text -->
<text
xml:space="preserve"
x="7200"
y="5175"
font-style="normal"
font-weight="normal"
font-size="324"
id="text24"
sodipodi:linespacing="125%"
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"><tspan
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"
id="tspan3027">Read-Write, Need Consistent Data</tspan></text>
<!-- Text -->
<text
xml:space="preserve"
x="7200"
y="6975"
font-style="normal"
font-weight="normal"
font-size="324"
id="text26"
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"
sodipodi:linespacing="125%">Update-Mostly, Need Consistent Data</text>
<!-- Text -->
<text
xml:space="preserve"
x="7200"
y="5625"
font-style="normal"
font-weight="normal"
font-size="324"
id="text28"
sodipodi:linespacing="125%"
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"><tspan
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"
id="tspan3029">(RCU Might Be OK...)</tspan></text>
<!-- Text -->
<text
xml:space="preserve"
x="7200"
y="7875"
font-style="normal"
font-weight="normal"
font-size="324"
id="text30"
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"
sodipodi:linespacing="125%">(1) Provide Existence Guarantees For Update-Friendly Mechanisms</text>
<!-- Text -->
<text
xml:space="preserve"
x="7200"
y="8325"
font-style="normal"
font-weight="normal"
font-size="324"
id="text32"
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"
sodipodi:linespacing="125%">(2) Provide Wait-Free Read-Side Primitives for Real-Time Use)</text>
<!-- Text -->
<text
xml:space="preserve"
x="7200"
y="7425"
font-style="normal"
font-weight="normal"
font-size="324"
id="text34"
style="font-size:427.63009644px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;font-family:Nimbus Sans L;-inkscape-font-specification:Nimbus Sans L"
sodipodi:linespacing="125%">(RCU is Very Unlikely to be the Right Tool For The Job, But it Can:</text>
</g>
</svg>
Documentation/RCU/Design/Requirements/ReadersPartitionGP1.svg 0 → 100644
View file @ 3104fb3d
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
width="735.25"
height="516.21875"
id="svg2"
version="1.1"
inkscape:version="0.48.3.1 r9886"
sodipodi:docname="ReadersPartitionGP1.svg">
<defs
id="defs4">
<marker
inkscape:stockid="Arrow2Lend"
orient="auto"
refY="0"
refX="0"
id="Arrow2Lend"
style="overflow:visible">
<path
id="path3792"
style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
inkscape:connector-curvature="0" />
</marker>
<marker
inkscape:stockid="Arrow2Lstart"
orient="auto"
refY="0"
refX="0"
id="Arrow2Lstart"
style="overflow:visible">
<path
id="path3789"
style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
transform="matrix(1.1,0,0,1.1,1.1,0)"
inkscape:connector-curvature="0" />
</marker>
<marker
inkscape:stockid="Arrow2Lstart"
orient="auto"
refY="0"
refX="0"
id="Arrow2Lstart-4"
style="overflow:visible">
<path
id="path3789-9"
style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
transform="matrix(1.1,0,0,1.1,1.1,0)"
inkscape:connector-curvature="0" />
</marker>
<marker
inkscape:stockid="Arrow2Lend"
orient="auto"
refY="0"
refX="0"
id="Arrow2Lend-4"
style="overflow:visible">
<path
id="path3792-4"
style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
inkscape:connector-curvature="0" />
</marker>
</defs>
<sodipodi:namedview
id="base"
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1.0"
inkscape:pageopacity="0.0"
inkscape:pageshadow="2"
inkscape:zoom="1.3670394"
inkscape:cx="367.26465"
inkscape:cy="258.46182"
inkscape:document-units="px"
inkscape:current-layer="g4433-6"
showgrid="false"
inkscape:window-width="1351"
inkscape:window-height="836"
inkscape:window-x="438"
inkscape:window-y="335"
inkscape:window-maximized="0"
fit-margin-top="5"
fit-margin-left="5"
fit-margin-right="5"
fit-margin-bottom="5" />
<metadata
id="metadata7">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title />
</cc:Work>
</rdf:RDF>
</metadata>
<g
inkscape:label="Layer 1"
inkscape:groupmode="layer"
id="layer1"
transform="translate(-29.15625,-185.59375)">
<flowRoot
xml:space="preserve"
id="flowRoot2985"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"><flowRegion
id="flowRegion2987"><rect
id="rect2989"
width="82.85714"
height="11.428572"
x="240"
y="492.36218" /></flowRegion><flowPara
id="flowPara2991" /></flowRoot> <g
id="g4433"
transform="translate(2,-12)">
<text
sodipodi:linespacing="125%"
id="text2993"
y="-261.66608"
x="436.12299"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
xml:space="preserve"
transform="matrix(0,1,-1,0,0,0)"><tspan
y="-261.66608"
x="436.12299"
id="tspan2995"
sodipodi:role="line">synchronize_rcu()</tspan></text>
<g
id="g4417"
transform="matrix(0,1,-1,0,730.90257,222.4928)">
<path
style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-start:url(#Arrow2Lstart);marker-end:url(#Arrow2Lend)"
d="M 97.580736,477.4048 327.57913,476.09759"
id="path2997"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
<path
style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
d="m 96.752718,465.38398 0,22.62742"
id="path4397"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
<path
style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
d="m 328.40703,465.38397 0,22.62742"
id="path4397-5"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
</g>
</g>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="112.04738"
y="268.18076"
id="text4429"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4431"
x="112.04738"
y="268.18076">WRITE_ONCE(a, 1);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="112.04738"
y="487.13766"
id="text4441"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4443"
x="112.04738"
y="487.13766">WRITE_ONCE(b, 1);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="255.60869"
y="297.29346"
id="text4445"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4447"
x="255.60869"
y="297.29346">r1 = READ_ONCE(a);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="255.14423"
y="554.61786"
id="text4449"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4451"
x="255.14423"
y="554.61786">WRITE_ONCE(c, 1);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="396.10254"
y="370.71124"
id="text4453"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4455"
x="396.10254"
y="370.71124">WRITE_ONCE(d, 1);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="396.10254"
y="572.13617"
id="text4457"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4459"
x="396.10254"
y="572.13617">r2 = READ_ONCE(c);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="112.08231"
y="213.91006"
id="text4461"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4463"
x="112.08231"
y="213.91006">thread0()</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="252.34512"
y="213.91006"
id="text4461-6"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4463-0"
x="252.34512"
y="213.91006">thread1()</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="396.42557"
y="213.91006"
id="text4461-2"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4463-2"
x="396.42557"
y="213.91006">thread2()</tspan></text>
<rect
style="fill:none;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
id="rect4495"
width="724.25244"
height="505.21201"
x="34.648232"
y="191.10612" />
<path
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
d="m 183.14066,191.10612 0,504.24243"
id="path4497"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
<path
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
d="m 325.13867,191.10612 0,504.24243"
id="path4497-5"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="111.75929"
y="251.53981"
id="text4429-8"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4431-9"
x="111.75929"
y="251.53981">rcu_read_lock();</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="396.10254"
y="353.91556"
id="text4429-8-9"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4431-9-4"
x="396.10254"
y="353.91556">rcu_read_lock();</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="396.10254"
y="587.40289"
id="text4429-8-9-3"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4431-9-4-4"
x="396.10254"
y="587.40289">rcu_read_unlock();</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="111.75929"
y="501.15311"
id="text4429-8-9-3-1"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4431-9-4-4-6"
x="111.75929"
y="501.15311">rcu_read_unlock();</tspan></text>
<path
style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
d="m 33.941125,227.87568 724.941765,0"
id="path4608"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="394.94427"
y="331.66351"
id="text4648"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4650"
x="394.94427"
y="331.66351">QS</tspan></text>
<path
sodipodi:type="arc"
style="fill:none;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
id="path4652"
sodipodi:cx="358.85669"
sodipodi:cy="142.87541"
sodipodi:rx="10.960155"
sodipodi:ry="10.253048"
d="m 358.86939,132.62237 a 10.960155,10.253048 0 1 1 -0.0228,0"
transform="translate(36.441125,185.60612)"
sodipodi:start="4.7135481"
sodipodi:end="10.994651"
sodipodi:open="true" />
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="112.11968"
y="523.77856"
id="text4648-4"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4650-4"
x="112.11968"
y="523.77856">QS</tspan></text>
<path
sodipodi:type="arc"
style="fill:none;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
id="path4652-7"
sodipodi:cx="358.85669"
sodipodi:cy="142.87541"
sodipodi:rx="10.960155"
sodipodi:ry="10.253048"
d="m 358.86939,132.62237 a 10.960155,10.253048 0 1 1 -0.0228,0"
transform="translate(-246.38346,377.72117)"
sodipodi:start="4.7135481"
sodipodi:end="10.994651"
sodipodi:open="true" />
<path
sodipodi:type="arc"
style="fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
id="path4652-7-7"
sodipodi:cx="358.85669"
sodipodi:cy="142.87541"
sodipodi:rx="10.960155"
sodipodi:ry="10.253048"
d="m 358.86939,132.62237 a 10.960155,10.253048 0 1 1 -0.0228,0"
transform="translate(-103.65246,190.90878)"
sodipodi:start="4.7135481"
sodipodi:end="10.994651"
sodipodi:open="true" />
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="254.85066"
y="336.96619"
id="text4648-4-3"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4650-4-5"
x="254.85066"
y="336.96619">QS</tspan></text>
<path
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
d="m 470.93311,190.39903 0,504.24243"
id="path4497-5-6"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
<path
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
d="m 616.22755,190.38323 0,504.24243"
id="path4497-5-2"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
<g
id="g4433-6"
transform="translate(288.0964,78.32827)">
<text
sodipodi:linespacing="125%"
id="text2993-7"
y="-261.66608"
x="440.12299"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
xml:space="preserve"
transform="matrix(0,1,-1,0,0,0)"><tspan
y="-261.66608"
x="440.12299"
id="tspan2995-1"
sodipodi:role="line">synchronize_rcu()</tspan></text>
<g
id="g4417-1"
transform="matrix(0,1,-1,0,730.90257,222.4928)">
<path
style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-start:url(#Arrow2Lstart);marker-end:url(#Arrow2Lend)"
d="M 97.580736,477.4048 328.5624,477.07246"
id="path2997-2"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
<path
style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
d="m 96.752718,465.38398 0,22.62742"
id="path4397-3"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
<path
style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
d="m 329.39039,465.38397 0,22.62742"
id="path4397-5-4"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cc" />
</g>
</g>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="541.70508"
y="387.6217"
id="text4445-0"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4447-5"
x="541.70508"
y="387.6217">r3 = READ_ONCE(d);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="541.2406"
y="646.94611"
id="text4449-6"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4451-6"
x="541.2406"
y="646.94611">WRITE_ONCE(e, 1);</tspan></text>
<path
sodipodi:type="arc"
style="fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
id="path4652-7-7-5"
sodipodi:cx="358.85669"
sodipodi:cy="142.87541"
sodipodi:rx="10.960155"
sodipodi:ry="10.253048"
d="m 358.86939,132.62237 a 10.960155,10.253048 0 1 1 -0.0228,0"
transform="translate(182.44393,281.23704)"
sodipodi:start="4.7135481"
sodipodi:end="10.994651"
sodipodi:open="true" />
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="540.94702"
y="427.29443"
id="text4648-4-3-1"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4650-4-5-7"
x="540.94702"
y="427.29443">QS</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="686.27747"
y="461.83929"
id="text4453-7"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4455-1"
x="686.27747"
y="461.83929">r4 = READ_ONCE(b);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="686.27747"
y="669.26422"
id="text4457-9"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4459-2"
x="686.27747"
y="669.26422">r5 = READ_ONCE(e);</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="686.27747"
y="445.04358"
id="text4429-8-9-33"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4431-9-4-2"
x="686.27747"
y="445.04358">rcu_read_lock();</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="686.27747"
y="684.53094"
id="text4429-8-9-3-8"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4431-9-4-4-5"
x="686.27747"
y="684.53094">rcu_read_unlock();</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="685.11914"
y="422.79153"
id="text4648-9"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4650-7"
x="685.11914"
y="422.79153">QS</tspan></text>
<path
sodipodi:type="arc"
style="fill:none;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
id="path4652-8"
sodipodi:cx="358.85669"
sodipodi:cy="142.87541"
sodipodi:rx="10.960155"
sodipodi:ry="10.253048"
d="m 358.86939,132.62237 a 10.960155,10.253048 0 1 1 -0.0228,0"
transform="translate(326.61602,276.73415)"
sodipodi:start="4.7135481"
sodipodi:end="10.994651"
sodipodi:open="true" />
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="397.85934"
y="609.59003"
id="text4648-5"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4650-77"
x="397.85934"
y="609.59003">QS</tspan></text>
<path
sodipodi:type="arc"
style="fill:none;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
id="path4652-80"
sodipodi:cx="358.85669"
sodipodi:cy="142.87541"
sodipodi:rx="10.960155"
sodipodi:ry="10.253048"
d="m 358.86939,132.62237 a 10.960155,10.253048 0 1 1 -0.0228,0"
transform="translate(39.356201,463.53264)"
sodipodi:start="4.7135481"
sodipodi:end="10.994651"
sodipodi:open="true" />
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="256.75986"
y="586.99133"
id="text4648-5-2"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4650-77-7"
x="256.75986"
y="586.99133">QS</tspan></text>
<path
sodipodi:type="arc"
style="fill:none;stroke:#000000;stroke-width:1;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
id="path4652-80-5"
sodipodi:cx="358.85669"
sodipodi:cy="142.87541"
sodipodi:rx="10.960155"
sodipodi:ry="10.253048"
d="m 358.86939,132.62237 a 10.960155,10.253048 0 1 1 -0.0228,0"
transform="translate(-101.74328,440.93395)"
sodipodi:start="4.7135481"
sodipodi:end="10.994651"
sodipodi:open="true" />
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="546.22791"
y="213.91006"
id="text4461-2-5"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4463-2-6"
x="546.22791"
y="213.91006">thread3()</tspan></text>
<text
xml:space="preserve"
style="font-size:10px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Symbol;-inkscape-font-specification:Symbol"
x="684.00067"
y="213.91006"
id="text4461-2-1"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan4463-2-0"
x="684.00067"
y="213.91006">thread4()</tspan></text>
</g>
</svg>
Documentation/RCU/Design/Requirements/Requirements.html 0 → 100644
View file @ 3104fb3d
This source diff could not be displayed because it is too large. You can view the blob instead.
Documentation/RCU/Design/Requirements/Requirements.htmlx 0 → 100644
View file @ 3104fb3d
This source diff could not be displayed because it is too large. You can view the blob instead.
Documentation/RCU/Design/htmlqqz.sh 0 → 100755
View file @ 3104fb3d
#!/bin/sh
#
# Usage: sh htmlqqz.sh file
#
# Extracts and converts quick quizzes in a proto-HTML document file.htmlx.
# Commands, all of which must be on a line by themselves:
#
# "<p>@@QQ@@": Start of a quick quiz.
# "<p>@@QQA@@": Start of a quick-quiz answer.
# "<p>@@QQE@@": End of a quick-quiz answer, and thus of the quick quiz.
# "<p>@@QQAL@@": Place to put quick-quiz answer list.
#
# Places the result in file.html.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# Copyright (c) 2013 Paul E. McKenney, IBM Corporation.
fn=$1
if test ! -r $fn.htmlx
then
echo "Error: $fn.htmlx unreadable."
exit 1
fi
echo "<!-- DO NOT HAND EDIT. -->" > $fn.html
echo "<!-- Instead, edit $fn.htmlx and run 'sh htmlqqz.sh $fn' -->" >> $fn.html
awk < $fn.htmlx >> $fn.html '
state == "" && $1 != "<p>@@QQ@@" && $1 != "<p>@@QQAL@@" {
print $0;
if ($0 ~ /^<p>@@QQ/)
print "Bad Quick Quiz command: " NR " (expected <p>@@QQ@@ or <p>@@QQAL@@)." > "/dev/stderr"
next;
}
state == "" && $1 == "<p>@@QQ@@" {
qqn++;
qqlineno = NR;
haveqq = 1;
state = "qq";
print "<p><a name=\"Quick Quiz " qqn "\"><b>Quick Quiz " qqn "</b>:</a>"
next;
}
state == "qq" && $1 != "<p>@@QQA@@" {
qq[qqn] = qq[qqn] $0 "\n";
print $0
if ($0 ~ /^<p>@@QQ/)
print "Bad Quick Quiz command: " NR ". (expected <p>@@QQA@@)" > "/dev/stderr"
next;
}
state == "qq" && $1 == "<p>@@QQA@@" {
state = "qqa";
print "<br><a href=\"#qq" qqn "answer\">Answer</a>"
next;
}
state == "qqa" && $1 != "<p>@@QQE@@" {
qqa[qqn] = qqa[qqn] $0 "\n";
if ($0 ~ /^<p>@@QQ/)
print "Bad Quick Quiz command: " NR " (expected <p>@@QQE@@)." > "/dev/stderr"
next;
}
state == "qqa" && $1 == "<p>@@QQE@@" {
state = "";
next;
}
state == "" && $1 == "<p>@@QQAL@@" {
haveqq = "";
print "<h3><a name=\"Answers to Quick Quizzes\">"
print "Answers to Quick Quizzes</a></h3>"
print "";
for (i = 1; i <= qqn; i++) {
print "<a name=\"qq" i "answer\"></a>"
print "<p><b>Quick Quiz " i "</b>:"
print qq[i];
print "";
print "</p><p><b>Answer</b>:"
print qqa[i];
print "";
print "</p><p><a href=\"#Quick%20Quiz%20" i "\"><b>Back to Quick Quiz " i "</b>.</a>"
print "";
}
next;
}
END {
if (state != "")
print "Unterminated Quick Quiz: " qqlineno "." > "/dev/stderr"
else if (haveqq)
print "Missing \"<p>@@QQAL@@\", no Quick Quiz." > "/dev/stderr"
}'
Documentation/kernel-parameters.txt
View file @ 3104fb3d
...@@ -3296,18 +3296,35 @@ bytes respectively. Such letter suffixes can also be entirely omitted. ...@@ -3296,18 +3296,35 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
rcutorture.verbose= [KNL] rcutorture.verbose= [KNL]
Enable additional printk() statements. Enable additional printk() statements.
rcupdate.rcu_cpu_stall_suppress= [KNL]
Suppress RCU CPU stall warning messages.
rcupdate.rcu_cpu_stall_timeout= [KNL]
Set timeout for RCU CPU stall warning messages.
rcupdate.rcu_expedited= [KNL] rcupdate.rcu_expedited= [KNL]
Use expedited grace-period primitives, for Use expedited grace-period primitives, for
example, synchronize_rcu_expedited() instead example, synchronize_rcu_expedited() instead
of synchronize_rcu(). This reduces latency, of synchronize_rcu(). This reduces latency,
but can increase CPU utilization, degrade but can increase CPU utilization, degrade
real-time latency, and degrade energy efficiency. real-time latency, and degrade energy efficiency.
No effect on CONFIG_TINY_RCU kernels.
rcupdate.rcu_cpu_stall_suppress= [KNL]
Suppress RCU CPU stall warning messages. rcupdate.rcu_normal= [KNL]
Use only normal grace-period primitives,
rcupdate.rcu_cpu_stall_timeout= [KNL] for example, synchronize_rcu() instead of
Set timeout for RCU CPU stall warning messages. synchronize_rcu_expedited(). This improves
real-time latency, CPU utilization, and
energy efficiency, but can expose users to
increased grace-period latency. This parameter
overrides rcupdate.rcu_expedited. No effect on
CONFIG_TINY_RCU kernels.
rcupdate.rcu_normal_after_boot= [KNL]
Once boot has completed (that is, after
rcu_end_inkernel_boot() has been invoked), use
only normal grace-period primitives. No effect
on CONFIG_TINY_RCU kernels.
rcupdate.rcu_task_stall_timeout= [KNL] rcupdate.rcu_task_stall_timeout= [KNL]
Set timeout in jiffies for RCU task stall warning Set timeout in jiffies for RCU task stall warning
......
Documentation/memory-barriers.txt
View file @ 3104fb3d
...@@ -194,7 +194,7 @@ There are some minimal guarantees that may be expected of a CPU: ...@@ -194,7 +194,7 @@ There are some minimal guarantees that may be expected of a CPU:
(*) On any given CPU, dependent memory accesses will be issued in order, with (*) On any given CPU, dependent memory accesses will be issued in order, with
respect to itself. This means that for: respect to itself. This means that for:
WRITE_ONCE(Q, P); smp_read_barrier_depends(); D = READ_ONCE(*Q); Q = READ_ONCE(P); smp_read_barrier_depends(); D = READ_ONCE(*Q);
the CPU will issue the following memory operations: the CPU will issue the following memory operations:
...@@ -202,9 +202,9 @@ There are some minimal guarantees that may be expected of a CPU: ...@@ -202,9 +202,9 @@ There are some minimal guarantees that may be expected of a CPU:
and always in that order. On most systems, smp_read_barrier_depends() and always in that order. On most systems, smp_read_barrier_depends()
does nothing, but it is required for DEC Alpha. The READ_ONCE() does nothing, but it is required for DEC Alpha. The READ_ONCE()
and WRITE_ONCE() are required to prevent compiler mischief. Please is required to prevent compiler mischief. Please note that you
note that you should normally use something like rcu_dereference() should normally use something like rcu_dereference() instead of
instead of open-coding smp_read_barrier_depends(). open-coding smp_read_barrier_depends().
(*) Overlapping loads and stores within a particular CPU will appear to be (*) Overlapping loads and stores within a particular CPU will appear to be
ordered within that CPU. This means that for: ordered within that CPU. This means that for:
......
drivers/tty/sysrq.c
View file @ 3104fb3d
...@@ -133,6 +133,12 @@ static void sysrq_handle_crash(int key) ...@@ -133,6 +133,12 @@ static void sysrq_handle_crash(int key)
{ {
char *killer = NULL; char *killer = NULL;
/* we need to release the RCU read lock here,
* otherwise we get an annoying
* 'BUG: sleeping function called from invalid context'
* complaint from the kernel before the panic.
*/
rcu_read_unlock();
panic_on_oops = 1; /* force panic */ panic_on_oops = 1; /* force panic */
wmb(); wmb();
*killer = 1; *killer = 1;
......
include/linux/list.h
View file @ 3104fb3d
...@@ -24,7 +24,7 @@ ...@@ -24,7 +24,7 @@
static inline void INIT_LIST_HEAD(struct list_head *list) static inline void INIT_LIST_HEAD(struct list_head *list)
{ {
list->next = list; WRITE_ONCE(list->next, list);
list->prev = list; list->prev = list;
} }
...@@ -42,7 +42,7 @@ static inline void __list_add(struct list_head *new, ...@@ -42,7 +42,7 @@ static inline void __list_add(struct list_head *new,
next->prev = new; next->prev = new;
new->next = next; new->next = next;
new->prev = prev; new->prev = prev;
prev->next = new; WRITE_ONCE(prev->next, new);
} }
#else #else
extern void __list_add(struct list_head *new, extern void __list_add(struct list_head *new,
...@@ -186,7 +186,7 @@ static inline int list_is_last(const struct list_head *list, ...@@ -186,7 +186,7 @@ static inline int list_is_last(const struct list_head *list,
*/ */
static inline int list_empty(const struct list_head *head) static inline int list_empty(const struct list_head *head)
{ {
return head->next == head; return READ_ONCE(head->next) == head;
} }
/** /**
...@@ -608,7 +608,7 @@ static inline int hlist_unhashed(const struct hlist_node *h) ...@@ -608,7 +608,7 @@ static inline int hlist_unhashed(const struct hlist_node *h)
static inline int hlist_empty(const struct hlist_head *h) static inline int hlist_empty(const struct hlist_head *h)
{ {
return !h->first; return !READ_ONCE(h->first);
} }
static inline void __hlist_del(struct hlist_node *n) static inline void __hlist_del(struct hlist_node *n)
...@@ -642,7 +642,7 @@ static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) ...@@ -642,7 +642,7 @@ static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
n->next = first; n->next = first;
if (first) if (first)
first->pprev = &n->next; first->pprev = &n->next;
h->first = n; WRITE_ONCE(h->first, n);
n->pprev = &h->first; n->pprev = &h->first;
} }
...@@ -653,14 +653,14 @@ static inline void hlist_add_before(struct hlist_node *n, ...@@ -653,14 +653,14 @@ static inline void hlist_add_before(struct hlist_node *n,
n->pprev = next->pprev; n->pprev = next->pprev;
n->next = next; n->next = next;
next->pprev = &n->next; next->pprev = &n->next;
*(n->pprev) = n; WRITE_ONCE(*(n->pprev), n);
} }
static inline void hlist_add_behind(struct hlist_node *n, static inline void hlist_add_behind(struct hlist_node *n,
struct hlist_node *prev) struct hlist_node *prev)
{ {
n->next = prev->next; n->next = prev->next;
prev->next = n; WRITE_ONCE(prev->next, n);
n->pprev = &prev->next; n->pprev = &prev->next;
if (n->next) if (n->next)
......
include/linux/list_bl.h
View file @ 3104fb3d
...@@ -70,7 +70,7 @@ static inline void hlist_bl_set_first(struct hlist_bl_head *h, ...@@ -70,7 +70,7 @@ static inline void hlist_bl_set_first(struct hlist_bl_head *h,
static inline int hlist_bl_empty(const struct hlist_bl_head *h) static inline int hlist_bl_empty(const struct hlist_bl_head *h)
{ {
return !((unsigned long)h->first & ~LIST_BL_LOCKMASK); return !((unsigned long)READ_ONCE(h->first) & ~LIST_BL_LOCKMASK);
} }
static inline void hlist_bl_add_head(struct hlist_bl_node *n, static inline void hlist_bl_add_head(struct hlist_bl_node *n,
......
include/linux/list_nulls.h
View file @ 3104fb3d
...@@ -57,7 +57,7 @@ static inline int hlist_nulls_unhashed(const struct hlist_nulls_node *h) ...@@ -57,7 +57,7 @@ static inline int hlist_nulls_unhashed(const struct hlist_nulls_node *h)
static inline int hlist_nulls_empty(const struct hlist_nulls_head *h) static inline int hlist_nulls_empty(const struct hlist_nulls_head *h)
{ {
return is_a_nulls(h->first); return is_a_nulls(READ_ONCE(h->first));
} }
static inline void hlist_nulls_add_head(struct hlist_nulls_node *n, static inline void hlist_nulls_add_head(struct hlist_nulls_node *n,
......
include/linux/rculist.h
View file @ 3104fb3d
...@@ -179,32 +179,31 @@ static inline void list_replace_rcu(struct list_head *old, ...@@ -179,32 +179,31 @@ static inline void list_replace_rcu(struct list_head *old,
} }
/** /**
* list_splice_init_rcu - splice an RCU-protected list into an existing list. * __list_splice_init_rcu - join an RCU-protected list into an existing list.
* @list: the RCU-protected list to splice * @list: the RCU-protected list to splice
* @head: the place in the list to splice the first list into * @prev: points to the last element of the existing list
* @next: points to the first element of the existing list
* @sync: function to sync: synchronize_rcu(), synchronize_sched(), ... * @sync: function to sync: synchronize_rcu(), synchronize_sched(), ...
* *
* @head can be RCU-read traversed concurrently with this function. * The list pointed to by @prev and @next can be RCU-read traversed
* concurrently with this function.
* *
* Note that this function blocks. * Note that this function blocks.
* *
* Important note: the caller must take whatever action is necessary to * Important note: the caller must take whatever action is necessary to prevent
* prevent any other updates to @head. In principle, it is possible * any other updates to the existing list. In principle, it is possible to
* to modify the list as soon as sync() begins execution. * modify the list as soon as sync() begins execution. If this sort of thing
* If this sort of thing becomes necessary, an alternative version * becomes necessary, an alternative version based on call_rcu() could be
* based on call_rcu() could be created. But only if -really- * created. But only if -really- needed -- there is no shortage of RCU API
* needed -- there is no shortage of RCU API members. * members.
*/ */
static inline void list_splice_init_rcu(struct list_head *list, static inline void __list_splice_init_rcu(struct list_head *list,
struct list_head *head, struct list_head *prev,
struct list_head *next,
void (*sync)(void)) void (*sync)(void))
{ {
struct list_head *first = list->next; struct list_head *first = list->next;
struct list_head *last = list->prev; struct list_head *last = list->prev;
struct list_head *at = head->next;
if (list_empty(list))
return;
/* /*
* "first" and "last" tracking list, so initialize it. RCU readers * "first" and "last" tracking list, so initialize it. RCU readers
...@@ -231,10 +230,40 @@ static inline void list_splice_init_rcu(struct list_head *list, ...@@ -231,10 +230,40 @@ static inline void list_splice_init_rcu(struct list_head *list,
* this function. * this function.
*/ */
last->next = at; last->next = next;
rcu_assign_pointer(list_next_rcu(head), first); rcu_assign_pointer(list_next_rcu(prev), first);
first->prev = head; first->prev = prev;
at->prev = last; next->prev = last;
}
/**
* list_splice_init_rcu - splice an RCU-protected list into an existing list,
* designed for stacks.
* @list: the RCU-protected list to splice
* @head: the place in the existing list to splice the first list into
* @sync: function to sync: synchronize_rcu(), synchronize_sched(), ...
*/
static inline void list_splice_init_rcu(struct list_head *list,
struct list_head *head,
void (*sync)(void))
{
if (!list_empty(list))
__list_splice_init_rcu(list, head, head->next, sync);
}
/**
* list_splice_tail_init_rcu - splice an RCU-protected list into an existing
* list, designed for queues.
* @list: the RCU-protected list to splice
* @head: the place in the existing list to splice the first list into
* @sync: function to sync: synchronize_rcu(), synchronize_sched(), ...
*/
static inline void list_splice_tail_init_rcu(struct list_head *list,
struct list_head *head,
void (*sync)(void))
{
if (!list_empty(list))
__list_splice_init_rcu(list, head->prev, head, sync);
} }
/** /**
...@@ -304,6 +333,42 @@ static inline void list_splice_init_rcu(struct list_head *list, ...@@ -304,6 +333,42 @@ static inline void list_splice_init_rcu(struct list_head *list,
&pos->member != (head); \ &pos->member != (head); \
pos = list_entry_rcu(pos->member.next, typeof(*pos), member)) pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
/**
* list_entry_lockless - get the struct for this entry
* @ptr: the &struct list_head pointer.
* @type: the type of the struct this is embedded in.
* @member: the name of the list_head within the struct.
*
* This primitive may safely run concurrently with the _rcu list-mutation
* primitives such as list_add_rcu(), but requires some implicit RCU
* read-side guarding. One example is running within a special
* exception-time environment where preemption is disabled and where
* lockdep cannot be invoked (in which case updaters must use RCU-sched,
* as in synchronize_sched(), call_rcu_sched(), and friends). Another
* example is when items are added to the list, but never deleted.
*/
#define list_entry_lockless(ptr, type, member) \
container_of((typeof(ptr))lockless_dereference(ptr), type, member)
/**
* list_for_each_entry_lockless - iterate over rcu list of given type
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_struct within the struct.
*
* This primitive may safely run concurrently with the _rcu list-mutation
* primitives such as list_add_rcu(), but requires some implicit RCU
* read-side guarding. One example is running within a special
* exception-time environment where preemption is disabled and where
* lockdep cannot be invoked (in which case updaters must use RCU-sched,
* as in synchronize_sched(), call_rcu_sched(), and friends). Another
* example is when items are added to the list, but never deleted.
*/
#define list_for_each_entry_lockless(pos, head, member) \
for (pos = list_entry_lockless((head)->next, typeof(*pos), member); \
&pos->member != (head); \
pos = list_entry_lockless(pos->member.next, typeof(*pos), member))
/** /**
* list_for_each_entry_continue_rcu - continue iteration over list of given type * list_for_each_entry_continue_rcu - continue iteration over list of given type
* @pos: the type * to use as a loop cursor. * @pos: the type * to use as a loop cursor.
......
include/linux/rcupdate.h
View file @ 3104fb3d
...@@ -48,10 +48,17 @@ ...@@ -48,10 +48,17 @@
#include <asm/barrier.h> #include <asm/barrier.h>
#ifndef CONFIG_TINY_RCU
extern int rcu_expedited; /* for sysctl */ extern int rcu_expedited; /* for sysctl */
extern int rcu_normal; /* also for sysctl */
#endif /* #ifndef CONFIG_TINY_RCU */
#ifdef CONFIG_TINY_RCU #ifdef CONFIG_TINY_RCU
/* Tiny RCU doesn't expedite, as its purpose in life is instead to be tiny. */ /* Tiny RCU doesn't expedite, as its purpose in life is instead to be tiny. */
static inline bool rcu_gp_is_normal(void) /* Internal RCU use. */
{
return true;
}
static inline bool rcu_gp_is_expedited(void) /* Internal RCU use. */ static inline bool rcu_gp_is_expedited(void) /* Internal RCU use. */
{ {
return false; return false;
...@@ -65,6 +72,7 @@ static inline void rcu_unexpedite_gp(void) ...@@ -65,6 +72,7 @@ static inline void rcu_unexpedite_gp(void)
{ {
} }
#else /* #ifdef CONFIG_TINY_RCU */ #else /* #ifdef CONFIG_TINY_RCU */
bool rcu_gp_is_normal(void); /* Internal RCU use. */
bool rcu_gp_is_expedited(void); /* Internal RCU use. */ bool rcu_gp_is_expedited(void); /* Internal RCU use. */
void rcu_expedite_gp(void); void rcu_expedite_gp(void);
void rcu_unexpedite_gp(void); void rcu_unexpedite_gp(void);
...@@ -321,7 +329,6 @@ static inline int rcu_preempt_depth(void) ...@@ -321,7 +329,6 @@ static inline int rcu_preempt_depth(void)
/* Internal to kernel */ /* Internal to kernel */
void rcu_init(void); void rcu_init(void);
void rcu_end_inkernel_boot(void);
void rcu_sched_qs(void); void rcu_sched_qs(void);
void rcu_bh_qs(void); void rcu_bh_qs(void);
void rcu_check_callbacks(int user); void rcu_check_callbacks(int user);
...@@ -329,6 +336,12 @@ struct notifier_block; ...@@ -329,6 +336,12 @@ struct notifier_block;
int rcu_cpu_notify(struct notifier_block *self, int rcu_cpu_notify(struct notifier_block *self,
unsigned long action, void *hcpu); unsigned long action, void *hcpu);
#ifndef CONFIG_TINY_RCU
void rcu_end_inkernel_boot(void);
#else /* #ifndef CONFIG_TINY_RCU */
static inline void rcu_end_inkernel_boot(void) { }
#endif /* #ifndef CONFIG_TINY_RCU */
#ifdef CONFIG_RCU_STALL_COMMON #ifdef CONFIG_RCU_STALL_COMMON
void rcu_sysrq_start(void); void rcu_sysrq_start(void);
void rcu_sysrq_end(void); void rcu_sysrq_end(void);
...@@ -379,9 +392,9 @@ static inline void rcu_init_nohz(void) ...@@ -379,9 +392,9 @@ static inline void rcu_init_nohz(void)
*/ */
#define RCU_NONIDLE(a) \ #define RCU_NONIDLE(a) \
do { \ do { \
rcu_irq_enter(); \ rcu_irq_enter_irqson(); \
do { a; } while (0); \ do { a; } while (0); \
rcu_irq_exit(); \ rcu_irq_exit_irqson(); \
} while (0) } while (0)
/* /*
...@@ -741,7 +754,7 @@ static inline void rcu_preempt_sleep_check(void) ...@@ -741,7 +754,7 @@ static inline void rcu_preempt_sleep_check(void)
* The tracing infrastructure traces RCU (we want that), but unfortunately * The tracing infrastructure traces RCU (we want that), but unfortunately
* some of the RCU checks causes tracing to lock up the system. * some of the RCU checks causes tracing to lock up the system.
* *
* The tracing version of rcu_dereference_raw() must not call * The no-tracing version of rcu_dereference_raw() must not call
* rcu_read_lock_held(). * rcu_read_lock_held().
*/ */
#define rcu_dereference_raw_notrace(p) __rcu_dereference_check((p), 1, __rcu) #define rcu_dereference_raw_notrace(p) __rcu_dereference_check((p), 1, __rcu)
......
include/linux/rcutiny.h
View file @ 3104fb3d
...@@ -181,6 +181,14 @@ static inline void rcu_irq_enter(void) ...@@ -181,6 +181,14 @@ static inline void rcu_irq_enter(void)
{ {
} }
static inline void rcu_irq_exit_irqson(void)
{
}
static inline void rcu_irq_enter_irqson(void)
{
}
static inline void rcu_irq_exit(void) static inline void rcu_irq_exit(void)
{ {
} }
......
include/linux/rcutree.h
View file @ 3104fb3d
...@@ -37,7 +37,7 @@ void rcu_cpu_stall_reset(void); ...@@ -37,7 +37,7 @@ void rcu_cpu_stall_reset(void);
/* /*
* Note a virtualization-based context switch. This is simply a * Note a virtualization-based context switch. This is simply a
* wrapper around rcu_note_context_switch(), which allows TINY_RCU * wrapper around rcu_note_context_switch(), which allows TINY_RCU
* to save a few bytes. * to save a few bytes. The caller must have disabled interrupts.
*/ */
static inline void rcu_virt_note_context_switch(int cpu) static inline void rcu_virt_note_context_switch(int cpu)
{ {
...@@ -97,6 +97,8 @@ void rcu_idle_enter(void); ...@@ -97,6 +97,8 @@ void rcu_idle_enter(void);
void rcu_idle_exit(void); void rcu_idle_exit(void);
void rcu_irq_enter(void); void rcu_irq_enter(void);
void rcu_irq_exit(void); void rcu_irq_exit(void);
void rcu_irq_enter_irqson(void);
void rcu_irq_exit_irqson(void);
void exit_rcu(void); void exit_rcu(void);
......
include/linux/tracepoint.h
View file @ 3104fb3d
...@@ -171,8 +171,8 @@ extern void syscall_unregfunc(void); ...@@ -171,8 +171,8 @@ extern void syscall_unregfunc(void);
TP_PROTO(data_proto), \ TP_PROTO(data_proto), \
TP_ARGS(data_args), \ TP_ARGS(data_args), \
TP_CONDITION(cond), \ TP_CONDITION(cond), \
rcu_irq_enter(), \ rcu_irq_enter_irqson(), \
rcu_irq_exit()); \ rcu_irq_exit_irqson()); \
} }
#else #else
#define __DECLARE_TRACE_RCU(name, proto, args, cond, data_proto, data_args) #define __DECLARE_TRACE_RCU(name, proto, args, cond, data_proto, data_args)
......
init/main.c
View file @ 3104fb3d
...@@ -943,6 +943,8 @@ static int __ref kernel_init(void *unused) ...@@ -943,6 +943,8 @@ static int __ref kernel_init(void *unused)
flush_delayed_fput(); flush_delayed_fput();
rcu_end_inkernel_boot();
if (ramdisk_execute_command) { if (ramdisk_execute_command) {
ret = run_init_process(ramdisk_execute_command); ret = run_init_process(ramdisk_execute_command);
if (!ret) if (!ret)
......
kernel/ksysfs.c
View file @ 3104fb3d
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
#include <linux/capability.h> #include <linux/capability.h>
#include <linux/compiler.h> #include <linux/compiler.h>
#include <linux/rcupdate.h> /* rcu_expedited */ #include <linux/rcupdate.h> /* rcu_expedited and rcu_normal */
#define KERNEL_ATTR_RO(_name) \ #define KERNEL_ATTR_RO(_name) \
static struct kobj_attribute _name##_attr = __ATTR_RO(_name) static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
...@@ -144,11 +144,12 @@ static ssize_t fscaps_show(struct kobject *kobj, ...@@ -144,11 +144,12 @@ static ssize_t fscaps_show(struct kobject *kobj,
} }
KERNEL_ATTR_RO(fscaps); KERNEL_ATTR_RO(fscaps);
#ifndef CONFIG_TINY_RCU
int rcu_expedited; int rcu_expedited;
static ssize_t rcu_expedited_show(struct kobject *kobj, static ssize_t rcu_expedited_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf) struct kobj_attribute *attr, char *buf)
{ {
return sprintf(buf, "%d\n", rcu_expedited); return sprintf(buf, "%d\n", READ_ONCE(rcu_expedited));
} }
static ssize_t rcu_expedited_store(struct kobject *kobj, static ssize_t rcu_expedited_store(struct kobject *kobj,
struct kobj_attribute *attr, struct kobj_attribute *attr,
...@@ -161,6 +162,24 @@ static ssize_t rcu_expedited_store(struct kobject *kobj, ...@@ -161,6 +162,24 @@ static ssize_t rcu_expedited_store(struct kobject *kobj,
} }
KERNEL_ATTR_RW(rcu_expedited); KERNEL_ATTR_RW(rcu_expedited);
int rcu_normal;
static ssize_t rcu_normal_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
return sprintf(buf, "%d\n", READ_ONCE(rcu_normal));
}
static ssize_t rcu_normal_store(struct kobject *kobj,
struct kobj_attribute *attr,
const char *buf, size_t count)
{
if (kstrtoint(buf, 0, &rcu_normal))
return -EINVAL;
return count;
}
KERNEL_ATTR_RW(rcu_normal);
#endif /* #ifndef CONFIG_TINY_RCU */
/* /*
* Make /sys/kernel/notes give the raw contents of our kernel .notes section. * Make /sys/kernel/notes give the raw contents of our kernel .notes section.
*/ */
...@@ -202,7 +221,10 @@ static struct attribute * kernel_attrs[] = { ...@@ -202,7 +221,10 @@ static struct attribute * kernel_attrs[] = {
&kexec_crash_size_attr.attr, &kexec_crash_size_attr.attr,
&vmcoreinfo_attr.attr, &vmcoreinfo_attr.attr,
#endif #endif
#ifndef CONFIG_TINY_RCU
&rcu_expedited_attr.attr, &rcu_expedited_attr.attr,
&rcu_normal_attr.attr,
#endif
NULL NULL
}; };
......
kernel/rcu/rcutorture.c
View file @ 3104fb3d
...@@ -162,6 +162,27 @@ static int rcu_torture_writer_state; ...@@ -162,6 +162,27 @@ static int rcu_torture_writer_state;
#define RTWS_SYNC 7 #define RTWS_SYNC 7
#define RTWS_STUTTER 8 #define RTWS_STUTTER 8
#define RTWS_STOPPING 9 #define RTWS_STOPPING 9
static const char * const rcu_torture_writer_state_names[] = {
"RTWS_FIXED_DELAY",
"RTWS_DELAY",
"RTWS_REPLACE",
"RTWS_DEF_FREE",
"RTWS_EXP_SYNC",
"RTWS_COND_GET",
"RTWS_COND_SYNC",
"RTWS_SYNC",
"RTWS_STUTTER",
"RTWS_STOPPING",
};
static const char *rcu_torture_writer_state_getname(void)
{
unsigned int i = READ_ONCE(rcu_torture_writer_state);
if (i >= ARRAY_SIZE(rcu_torture_writer_state_names))
return "???";
return rcu_torture_writer_state_names[i];
}
#if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE) #if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
#define RCUTORTURE_RUNNABLE_INIT 1 #define RCUTORTURE_RUNNABLE_INIT 1
...@@ -1307,7 +1328,8 @@ rcu_torture_stats_print(void) ...@@ -1307,7 +1328,8 @@ rcu_torture_stats_print(void)
rcutorture_get_gp_data(cur_ops->ttype, rcutorture_get_gp_data(cur_ops->ttype,
&flags, &gpnum, &completed); &flags, &gpnum, &completed);
pr_alert("??? Writer stall state %d g%lu c%lu f%#x\n", pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x\n",
rcu_torture_writer_state_getname(),
rcu_torture_writer_state, rcu_torture_writer_state,
gpnum, completed, flags); gpnum, completed, flags);
show_rcu_gp_kthreads(); show_rcu_gp_kthreads();
......
kernel/rcu/srcu.c
View file @ 3104fb3d
...@@ -489,7 +489,7 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount) ...@@ -489,7 +489,7 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
*/ */
void synchronize_srcu(struct srcu_struct *sp) void synchronize_srcu(struct srcu_struct *sp)
{ {
__synchronize_srcu(sp, rcu_gp_is_expedited() __synchronize_srcu(sp, (rcu_gp_is_expedited() && !rcu_gp_is_normal())
? SYNCHRONIZE_SRCU_EXP_TRYCOUNT ? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
: SYNCHRONIZE_SRCU_TRYCOUNT); : SYNCHRONIZE_SRCU_TRYCOUNT);
} }
......
kernel/rcu/tree.c
View file @ 3104fb3d
...@@ -68,10 +68,6 @@ MODULE_ALIAS("rcutree"); ...@@ -68,10 +68,6 @@ MODULE_ALIAS("rcutree");
/* Data structures. */ /* Data structures. */
static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
static struct lock_class_key rcu_exp_class[RCU_NUM_LVLS];
/* /*
* In order to export the rcu_state name to the tracing tools, it * In order to export the rcu_state name to the tracing tools, it
* needs to be added in the __tracepoint_string section. * needs to be added in the __tracepoint_string section.
...@@ -246,24 +242,17 @@ static int rcu_gp_in_progress(struct rcu_state *rsp) ...@@ -246,24 +242,17 @@ static int rcu_gp_in_progress(struct rcu_state *rsp)
*/ */
void rcu_sched_qs(void) void rcu_sched_qs(void)
{ {
unsigned long flags; if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.s))
return;
if (__this_cpu_read(rcu_sched_data.cpu_no_qs.s)) {
trace_rcu_grace_period(TPS("rcu_sched"), trace_rcu_grace_period(TPS("rcu_sched"),
__this_cpu_read(rcu_sched_data.gpnum), __this_cpu_read(rcu_sched_data.gpnum),
TPS("cpuqs")); TPS("cpuqs"));
__this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false); __this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);
if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp)) if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
return; return;
local_irq_save(flags);
if (__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp)) {
__this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, false); __this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, false);
rcu_report_exp_rdp(&rcu_sched_state, rcu_report_exp_rdp(&rcu_sched_state,
this_cpu_ptr(&rcu_sched_data), this_cpu_ptr(&rcu_sched_data), true);
true);
}
local_irq_restore(flags);
}
} }
void rcu_bh_qs(void) void rcu_bh_qs(void)
...@@ -300,17 +289,16 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr); ...@@ -300,17 +289,16 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
* We inform the RCU core by emulating a zero-duration dyntick-idle * We inform the RCU core by emulating a zero-duration dyntick-idle
* period, which we in turn do by incrementing the ->dynticks counter * period, which we in turn do by incrementing the ->dynticks counter
* by two. * by two.
*
* The caller must have disabled interrupts.
*/ */
static void rcu_momentary_dyntick_idle(void) static void rcu_momentary_dyntick_idle(void)
{ {
unsigned long flags;
struct rcu_data *rdp; struct rcu_data *rdp;
struct rcu_dynticks *rdtp; struct rcu_dynticks *rdtp;
int resched_mask; int resched_mask;
struct rcu_state *rsp; struct rcu_state *rsp;
local_irq_save(flags);
/* /*
* Yes, we can lose flag-setting operations. This is OK, because * Yes, we can lose flag-setting operations. This is OK, because
* the flag will be set again after some delay. * the flag will be set again after some delay.
...@@ -340,13 +328,12 @@ static void rcu_momentary_dyntick_idle(void) ...@@ -340,13 +328,12 @@ static void rcu_momentary_dyntick_idle(void)
smp_mb__after_atomic(); /* Later stuff after QS. */ smp_mb__after_atomic(); /* Later stuff after QS. */
break; break;
} }
local_irq_restore(flags);
} }
/* /*
* Note a context switch. This is a quiescent state for RCU-sched, * Note a context switch. This is a quiescent state for RCU-sched,
* and requires special handling for preemptible RCU. * and requires special handling for preemptible RCU.
* The caller must have disabled preemption. * The caller must have disabled interrupts.
*/ */
void rcu_note_context_switch(void) void rcu_note_context_switch(void)
{ {
...@@ -376,9 +363,14 @@ EXPORT_SYMBOL_GPL(rcu_note_context_switch); ...@@ -376,9 +363,14 @@ EXPORT_SYMBOL_GPL(rcu_note_context_switch);
*/ */
void rcu_all_qs(void) void rcu_all_qs(void)
{ {
unsigned long flags;
barrier(); /* Avoid RCU read-side critical sections leaking down. */ barrier(); /* Avoid RCU read-side critical sections leaking down. */
if (unlikely(raw_cpu_read(rcu_sched_qs_mask))) if (unlikely(raw_cpu_read(rcu_sched_qs_mask))) {
local_irq_save(flags);
rcu_momentary_dyntick_idle(); rcu_momentary_dyntick_idle();
local_irq_restore(flags);
}
this_cpu_inc(rcu_qs_ctr); this_cpu_inc(rcu_qs_ctr);
barrier(); /* Avoid RCU read-side critical sections leaking up. */ barrier(); /* Avoid RCU read-side critical sections leaking up. */
} }
...@@ -605,25 +597,25 @@ static int rcu_future_needs_gp(struct rcu_state *rsp) ...@@ -605,25 +597,25 @@ static int rcu_future_needs_gp(struct rcu_state *rsp)
* The caller must have disabled interrupts to prevent races with * The caller must have disabled interrupts to prevent races with
* normal callback registry. * normal callback registry.
*/ */
static int static bool
cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp)
{ {
int i; int i;
if (rcu_gp_in_progress(rsp)) if (rcu_gp_in_progress(rsp))
return 0; /* No, a grace period is already in progress. */ return false; /* No, a grace period is already in progress. */
if (rcu_future_needs_gp(rsp)) if (rcu_future_needs_gp(rsp))
return 1; /* Yes, a no-CBs CPU needs one. */ return true; /* Yes, a no-CBs CPU needs one. */
if (!rdp->nxttail[RCU_NEXT_TAIL]) if (!rdp->nxttail[RCU_NEXT_TAIL])
return 0; /* No, this is a no-CBs (or offline) CPU. */ return false; /* No, this is a no-CBs (or offline) CPU. */
if (*rdp->nxttail[RCU_NEXT_READY_TAIL]) if (*rdp->nxttail[RCU_NEXT_READY_TAIL])
return 1; /* Yes, this CPU has newly registered callbacks. */ return true; /* Yes, CPU has newly registered callbacks. */
for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++)
if (rdp->nxttail[i - 1] != rdp->nxttail[i] && if (rdp->nxttail[i - 1] != rdp->nxttail[i] &&
ULONG_CMP_LT(READ_ONCE(rsp->completed), ULONG_CMP_LT(READ_ONCE(rsp->completed),
rdp->nxtcompleted[i])) rdp->nxtcompleted[i]))
return 1; /* Yes, CBs for future grace period. */ return true; /* Yes, CBs for future grace period. */
return 0; /* No grace period needed. */ return false; /* No grace period needed. */
} }
/* /*
...@@ -740,7 +732,7 @@ void rcu_user_enter(void) ...@@ -740,7 +732,7 @@ void rcu_user_enter(void)
* *
* Exit from an interrupt handler, which might possibly result in entering * Exit from an interrupt handler, which might possibly result in entering
* idle mode, in other words, leaving the mode in which read-side critical * idle mode, in other words, leaving the mode in which read-side critical
* sections can occur. * sections can occur. The caller must have disabled interrupts.
* *
* This code assumes that the idle loop never does anything that might * This code assumes that the idle loop never does anything that might
* result in unbalanced calls to irq_enter() and irq_exit(). If your * result in unbalanced calls to irq_enter() and irq_exit(). If your
...@@ -753,11 +745,10 @@ void rcu_user_enter(void) ...@@ -753,11 +745,10 @@ void rcu_user_enter(void)
*/ */
void rcu_irq_exit(void) void rcu_irq_exit(void)
{ {
unsigned long flags;
long long oldval; long long oldval;
struct rcu_dynticks *rdtp; struct rcu_dynticks *rdtp;
local_irq_save(flags); RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_irq_exit() invoked with irqs enabled!!!");
rdtp = this_cpu_ptr(&rcu_dynticks); rdtp = this_cpu_ptr(&rcu_dynticks);
oldval = rdtp->dynticks_nesting; oldval = rdtp->dynticks_nesting;
rdtp->dynticks_nesting--; rdtp->dynticks_nesting--;
...@@ -768,6 +759,17 @@ void rcu_irq_exit(void) ...@@ -768,6 +759,17 @@ void rcu_irq_exit(void)
else else
rcu_eqs_enter_common(oldval, true); rcu_eqs_enter_common(oldval, true);
rcu_sysidle_enter(1); rcu_sysidle_enter(1);
}
/*
* Wrapper for rcu_irq_exit() where interrupts are enabled.
*/
void rcu_irq_exit_irqson(void)
{
unsigned long flags;
local_irq_save(flags);
rcu_irq_exit();
local_irq_restore(flags); local_irq_restore(flags);
} }
...@@ -865,7 +867,7 @@ void rcu_user_exit(void) ...@@ -865,7 +867,7 @@ void rcu_user_exit(void)
* *
* Enter an interrupt handler, which might possibly result in exiting * Enter an interrupt handler, which might possibly result in exiting
* idle mode, in other words, entering the mode in which read-side critical * idle mode, in other words, entering the mode in which read-side critical
* sections can occur. * sections can occur. The caller must have disabled interrupts.
* *
* Note that the Linux kernel is fully capable of entering an interrupt * Note that the Linux kernel is fully capable of entering an interrupt
* handler that it never exits, for example when doing upcalls to * handler that it never exits, for example when doing upcalls to
...@@ -881,11 +883,10 @@ void rcu_user_exit(void) ...@@ -881,11 +883,10 @@ void rcu_user_exit(void)
*/ */
void rcu_irq_enter(void) void rcu_irq_enter(void)
{ {
unsigned long flags;
struct rcu_dynticks *rdtp; struct rcu_dynticks *rdtp;
long long oldval; long long oldval;
local_irq_save(flags); RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_irq_enter() invoked with irqs enabled!!!");
rdtp = this_cpu_ptr(&rcu_dynticks); rdtp = this_cpu_ptr(&rcu_dynticks);
oldval = rdtp->dynticks_nesting; oldval = rdtp->dynticks_nesting;
rdtp->dynticks_nesting++; rdtp->dynticks_nesting++;
...@@ -896,6 +897,17 @@ void rcu_irq_enter(void) ...@@ -896,6 +897,17 @@ void rcu_irq_enter(void)
else else
rcu_eqs_exit_common(oldval, true); rcu_eqs_exit_common(oldval, true);
rcu_sysidle_exit(1); rcu_sysidle_exit(1);
}
/*
* Wrapper for rcu_irq_enter() where interrupts are enabled.
*/
void rcu_irq_enter_irqson(void)
{
unsigned long flags;
local_irq_save(flags);
rcu_irq_enter();
local_irq_restore(flags); local_irq_restore(flags);
} }
...@@ -1186,6 +1198,16 @@ static void record_gp_stall_check_time(struct rcu_state *rsp) ...@@ -1186,6 +1198,16 @@ static void record_gp_stall_check_time(struct rcu_state *rsp)
rsp->n_force_qs_gpstart = READ_ONCE(rsp->n_force_qs); rsp->n_force_qs_gpstart = READ_ONCE(rsp->n_force_qs);
} }
/*
* Convert a ->gp_state value to a character string.
*/
static const char *gp_state_getname(short gs)
{
if (gs < 0 || gs >= ARRAY_SIZE(gp_state_names))
return "???";
return gp_state_names[gs];
}
/* /*
* Complain about starvation of grace-period kthread. * Complain about starvation of grace-period kthread.
*/ */
...@@ -1196,12 +1218,16 @@ static void rcu_check_gp_kthread_starvation(struct rcu_state *rsp) ...@@ -1196,12 +1218,16 @@ static void rcu_check_gp_kthread_starvation(struct rcu_state *rsp)
j = jiffies; j = jiffies;
gpa = READ_ONCE(rsp->gp_activity); gpa = READ_ONCE(rsp->gp_activity);
if (j - gpa > 2 * HZ) if (j - gpa > 2 * HZ) {
pr_err("%s kthread starved for %ld jiffies! g%lu c%lu f%#x s%d ->state=%#lx\n", pr_err("%s kthread starved for %ld jiffies! g%lu c%lu f%#x %s(%d) ->state=%#lx\n",
rsp->name, j - gpa, rsp->name, j - gpa,
rsp->gpnum, rsp->completed, rsp->gpnum, rsp->completed,
rsp->gp_flags, rsp->gp_state, rsp->gp_flags,
rsp->gp_kthread ? rsp->gp_kthread->state : 0); gp_state_getname(rsp->gp_state), rsp->gp_state,
rsp->gp_kthread ? rsp->gp_kthread->state : ~0);
if (rsp->gp_kthread)
sched_show_task(rsp->gp_kthread);
}
} }
/* /*
...@@ -1214,7 +1240,7 @@ static void rcu_dump_cpu_stacks(struct rcu_state *rsp) ...@@ -1214,7 +1240,7 @@ static void rcu_dump_cpu_stacks(struct rcu_state *rsp)
struct rcu_node *rnp; struct rcu_node *rnp;
rcu_for_each_leaf_node(rsp, rnp) { rcu_for_each_leaf_node(rsp, rnp) {
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
if (rnp->qsmask != 0) { if (rnp->qsmask != 0) {
for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++) for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++)
if (rnp->qsmask & (1UL << cpu)) if (rnp->qsmask & (1UL << cpu))
...@@ -1237,7 +1263,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum) ...@@ -1237,7 +1263,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)
/* Only let one CPU complain about others per time interval. */ /* Only let one CPU complain about others per time interval. */
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
delta = jiffies - READ_ONCE(rsp->jiffies_stall); delta = jiffies - READ_ONCE(rsp->jiffies_stall);
if (delta < RCU_STALL_RAT_DELAY || !rcu_gp_in_progress(rsp)) { if (delta < RCU_STALL_RAT_DELAY || !rcu_gp_in_progress(rsp)) {
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
...@@ -1256,7 +1282,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum) ...@@ -1256,7 +1282,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)
rsp->name); rsp->name);
print_cpu_stall_info_begin(); print_cpu_stall_info_begin();
rcu_for_each_leaf_node(rsp, rnp) { rcu_for_each_leaf_node(rsp, rnp) {
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
ndetected += rcu_print_task_stall(rnp); ndetected += rcu_print_task_stall(rnp);
if (rnp->qsmask != 0) { if (rnp->qsmask != 0) {
for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++) for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++)
...@@ -1327,7 +1353,7 @@ static void print_cpu_stall(struct rcu_state *rsp) ...@@ -1327,7 +1353,7 @@ static void print_cpu_stall(struct rcu_state *rsp)
rcu_dump_cpu_stacks(rsp); rcu_dump_cpu_stacks(rsp);
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall))) if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall)))
WRITE_ONCE(rsp->jiffies_stall, WRITE_ONCE(rsp->jiffies_stall,
jiffies + 3 * rcu_jiffies_till_stall_check() + 3); jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
...@@ -1534,10 +1560,8 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp, ...@@ -1534,10 +1560,8 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp,
* hold it, acquire the root rcu_node structure's lock in order to * hold it, acquire the root rcu_node structure's lock in order to
* start one (if needed). * start one (if needed).
*/ */
if (rnp != rnp_root) { if (rnp != rnp_root)
raw_spin_lock(&rnp_root->lock); raw_spin_lock_rcu_node(rnp_root);
smp_mb__after_unlock_lock();
}
/* /*
* Get a new grace-period number. If there really is no grace * Get a new grace-period number. If there really is no grace
...@@ -1786,11 +1810,10 @@ static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp) ...@@ -1786,11 +1810,10 @@ static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp)
if ((rdp->gpnum == READ_ONCE(rnp->gpnum) && if ((rdp->gpnum == READ_ONCE(rnp->gpnum) &&
rdp->completed == READ_ONCE(rnp->completed) && rdp->completed == READ_ONCE(rnp->completed) &&
!unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */ !unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */
!raw_spin_trylock(&rnp->lock)) { /* irqs already off, so later. */ !raw_spin_trylock_rcu_node(rnp)) { /* irqs already off, so later. */
local_irq_restore(flags); local_irq_restore(flags);
return; return;
} }
smp_mb__after_unlock_lock();
needwake = __note_gp_changes(rsp, rnp, rdp); needwake = __note_gp_changes(rsp, rnp, rdp);
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
if (needwake) if (needwake)
...@@ -1805,21 +1828,20 @@ static void rcu_gp_slow(struct rcu_state *rsp, int delay) ...@@ -1805,21 +1828,20 @@ static void rcu_gp_slow(struct rcu_state *rsp, int delay)
} }
/* /*
* Initialize a new grace period. Return 0 if no grace period required. * Initialize a new grace period. Return false if no grace period required.
*/ */
static int rcu_gp_init(struct rcu_state *rsp) static bool rcu_gp_init(struct rcu_state *rsp)
{ {
unsigned long oldmask; unsigned long oldmask;
struct rcu_data *rdp; struct rcu_data *rdp;
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
WRITE_ONCE(rsp->gp_activity, jiffies); WRITE_ONCE(rsp->gp_activity, jiffies);
raw_spin_lock_irq(&rnp->lock); raw_spin_lock_irq_rcu_node(rnp);
smp_mb__after_unlock_lock();
if (!READ_ONCE(rsp->gp_flags)) { if (!READ_ONCE(rsp->gp_flags)) {
/* Spurious wakeup, tell caller to go back to sleep. */ /* Spurious wakeup, tell caller to go back to sleep. */
raw_spin_unlock_irq(&rnp->lock); raw_spin_unlock_irq(&rnp->lock);
return 0; return false;
} }
WRITE_ONCE(rsp->gp_flags, 0); /* Clear all flags: New grace period. */ WRITE_ONCE(rsp->gp_flags, 0); /* Clear all flags: New grace period. */
...@@ -1829,7 +1851,7 @@ static int rcu_gp_init(struct rcu_state *rsp) ...@@ -1829,7 +1851,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
* Not supposed to be able to happen. * Not supposed to be able to happen.
*/ */
raw_spin_unlock_irq(&rnp->lock); raw_spin_unlock_irq(&rnp->lock);
return 0; return false;
} }
/* Advance to a new grace period and initialize state. */ /* Advance to a new grace period and initialize state. */
...@@ -1847,8 +1869,7 @@ static int rcu_gp_init(struct rcu_state *rsp) ...@@ -1847,8 +1869,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
*/ */
rcu_for_each_leaf_node(rsp, rnp) { rcu_for_each_leaf_node(rsp, rnp) {
rcu_gp_slow(rsp, gp_preinit_delay); rcu_gp_slow(rsp, gp_preinit_delay);
raw_spin_lock_irq(&rnp->lock); raw_spin_lock_irq_rcu_node(rnp);
smp_mb__after_unlock_lock();
if (rnp->qsmaskinit == rnp->qsmaskinitnext && if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
!rnp->wait_blkd_tasks) { !rnp->wait_blkd_tasks) {
/* Nothing to do on this leaf rcu_node structure. */ /* Nothing to do on this leaf rcu_node structure. */
...@@ -1904,8 +1925,7 @@ static int rcu_gp_init(struct rcu_state *rsp) ...@@ -1904,8 +1925,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
*/ */
rcu_for_each_node_breadth_first(rsp, rnp) { rcu_for_each_node_breadth_first(rsp, rnp) {
rcu_gp_slow(rsp, gp_init_delay); rcu_gp_slow(rsp, gp_init_delay);
raw_spin_lock_irq(&rnp->lock); raw_spin_lock_irq_rcu_node(rnp);
smp_mb__after_unlock_lock();
rdp = this_cpu_ptr(rsp->rda); rdp = this_cpu_ptr(rsp->rda);
rcu_preempt_check_blocked_tasks(rnp); rcu_preempt_check_blocked_tasks(rnp);
rnp->qsmask = rnp->qsmaskinit; rnp->qsmask = rnp->qsmaskinit;
...@@ -1923,7 +1943,7 @@ static int rcu_gp_init(struct rcu_state *rsp) ...@@ -1923,7 +1943,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
WRITE_ONCE(rsp->gp_activity, jiffies); WRITE_ONCE(rsp->gp_activity, jiffies);
} }
return 1; return true;
} }
/* /*
...@@ -1973,8 +1993,7 @@ static void rcu_gp_fqs(struct rcu_state *rsp, bool first_time) ...@@ -1973,8 +1993,7 @@ static void rcu_gp_fqs(struct rcu_state *rsp, bool first_time)
} }
/* Clear flag to prevent immediate re-entry. */ /* Clear flag to prevent immediate re-entry. */
if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
raw_spin_lock_irq(&rnp->lock); raw_spin_lock_irq_rcu_node(rnp);
smp_mb__after_unlock_lock();
WRITE_ONCE(rsp->gp_flags, WRITE_ONCE(rsp->gp_flags,
READ_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS); READ_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS);
raw_spin_unlock_irq(&rnp->lock); raw_spin_unlock_irq(&rnp->lock);
...@@ -1993,8 +2012,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) ...@@ -1993,8 +2012,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
WRITE_ONCE(rsp->gp_activity, jiffies); WRITE_ONCE(rsp->gp_activity, jiffies);
raw_spin_lock_irq(&rnp->lock); raw_spin_lock_irq_rcu_node(rnp);
smp_mb__after_unlock_lock();
gp_duration = jiffies - rsp->gp_start; gp_duration = jiffies - rsp->gp_start;
if (gp_duration > rsp->gp_max) if (gp_duration > rsp->gp_max)
rsp->gp_max = gp_duration; rsp->gp_max = gp_duration;
...@@ -2019,8 +2037,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) ...@@ -2019,8 +2037,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
* grace period is recorded in any of the rcu_node structures. * grace period is recorded in any of the rcu_node structures.
*/ */
rcu_for_each_node_breadth_first(rsp, rnp) { rcu_for_each_node_breadth_first(rsp, rnp) {
raw_spin_lock_irq(&rnp->lock); raw_spin_lock_irq_rcu_node(rnp);
smp_mb__after_unlock_lock();
WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)); WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
WARN_ON_ONCE(rnp->qsmask); WARN_ON_ONCE(rnp->qsmask);
WRITE_ONCE(rnp->completed, rsp->gpnum); WRITE_ONCE(rnp->completed, rsp->gpnum);
...@@ -2035,8 +2052,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) ...@@ -2035,8 +2052,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
rcu_gp_slow(rsp, gp_cleanup_delay); rcu_gp_slow(rsp, gp_cleanup_delay);
} }
rnp = rcu_get_root(rsp); rnp = rcu_get_root(rsp);
raw_spin_lock_irq(&rnp->lock); raw_spin_lock_irq_rcu_node(rnp); /* Order GP before ->completed update. */
smp_mb__after_unlock_lock(); /* Order GP before ->completed update. */
rcu_nocb_gp_set(rnp, nocb); rcu_nocb_gp_set(rnp, nocb);
/* Declare grace period done. */ /* Declare grace period done. */
...@@ -2284,8 +2300,7 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp, ...@@ -2284,8 +2300,7 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
rnp_c = rnp; rnp_c = rnp;
rnp = rnp->parent; rnp = rnp->parent;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
smp_mb__after_unlock_lock();
oldmask = rnp_c->qsmask; oldmask = rnp_c->qsmask;
} }
...@@ -2332,8 +2347,7 @@ static void rcu_report_unblock_qs_rnp(struct rcu_state *rsp, ...@@ -2332,8 +2347,7 @@ static void rcu_report_unblock_qs_rnp(struct rcu_state *rsp,
gps = rnp->gpnum; gps = rnp->gpnum;
mask = rnp->grpmask; mask = rnp->grpmask;
raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */ raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
raw_spin_lock(&rnp_p->lock); /* irqs already disabled. */ raw_spin_lock_rcu_node(rnp_p); /* irqs already disabled. */
smp_mb__after_unlock_lock();
rcu_report_qs_rnp(mask, rsp, rnp_p, gps, flags); rcu_report_qs_rnp(mask, rsp, rnp_p, gps, flags);
} }
...@@ -2355,8 +2369,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) ...@@ -2355,8 +2369,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
struct rcu_node *rnp; struct rcu_node *rnp;
rnp = rdp->mynode; rnp = rdp->mynode;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
smp_mb__after_unlock_lock();
if ((rdp->cpu_no_qs.b.norm && if ((rdp->cpu_no_qs.b.norm &&
rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) || rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||
rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum || rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||
...@@ -2582,8 +2595,7 @@ static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf) ...@@ -2582,8 +2595,7 @@ static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
rnp = rnp->parent; rnp = rnp->parent;
if (!rnp) if (!rnp)
break; break;
raw_spin_lock(&rnp->lock); /* irqs already disabled. */ raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
smp_mb__after_unlock_lock(); /* GP memory ordering. */
rnp->qsmaskinit &= ~mask; rnp->qsmaskinit &= ~mask;
rnp->qsmask &= ~mask; rnp->qsmask &= ~mask;
if (rnp->qsmaskinit) { if (rnp->qsmaskinit) {
...@@ -2611,8 +2623,7 @@ static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp) ...@@ -2611,8 +2623,7 @@ static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp)
/* Remove outgoing CPU from mask in the leaf rcu_node structure. */ /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
mask = rdp->grpmask; mask = rdp->grpmask;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
smp_mb__after_unlock_lock(); /* Enforce GP memory-order guarantee. */
rnp->qsmaskinitnext &= ~mask; rnp->qsmaskinitnext &= ~mask;
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
} }
...@@ -2809,8 +2820,7 @@ static void force_qs_rnp(struct rcu_state *rsp, ...@@ -2809,8 +2820,7 @@ static void force_qs_rnp(struct rcu_state *rsp,
rcu_for_each_leaf_node(rsp, rnp) { rcu_for_each_leaf_node(rsp, rnp) {
cond_resched_rcu_qs(); cond_resched_rcu_qs();
mask = 0; mask = 0;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
smp_mb__after_unlock_lock();
if (rnp->qsmask == 0) { if (rnp->qsmask == 0) {
if (rcu_state_p == &rcu_sched_state || if (rcu_state_p == &rcu_sched_state ||
rsp != rcu_state_p || rsp != rcu_state_p ||
...@@ -2881,8 +2891,7 @@ static void force_quiescent_state(struct rcu_state *rsp) ...@@ -2881,8 +2891,7 @@ static void force_quiescent_state(struct rcu_state *rsp)
/* rnp_old == rcu_get_root(rsp), rnp == NULL. */ /* rnp_old == rcu_get_root(rsp), rnp == NULL. */
/* Reached the root of the rcu_node tree, acquire lock. */ /* Reached the root of the rcu_node tree, acquire lock. */
raw_spin_lock_irqsave(&rnp_old->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp_old, flags);
smp_mb__after_unlock_lock();
raw_spin_unlock(&rnp_old->fqslock); raw_spin_unlock(&rnp_old->fqslock);
if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
rsp->n_force_qs_lh++; rsp->n_force_qs_lh++;
...@@ -2914,7 +2923,7 @@ __rcu_process_callbacks(struct rcu_state *rsp) ...@@ -2914,7 +2923,7 @@ __rcu_process_callbacks(struct rcu_state *rsp)
/* Does this CPU require a not-yet-started grace period? */ /* Does this CPU require a not-yet-started grace period? */
local_irq_save(flags); local_irq_save(flags);
if (cpu_needs_another_gp(rsp, rdp)) { if (cpu_needs_another_gp(rsp, rdp)) {
raw_spin_lock(&rcu_get_root(rsp)->lock); /* irqs disabled. */ raw_spin_lock_rcu_node(rcu_get_root(rsp)); /* irqs disabled. */
needwake = rcu_start_gp(rsp); needwake = rcu_start_gp(rsp);
raw_spin_unlock_irqrestore(&rcu_get_root(rsp)->lock, flags); raw_spin_unlock_irqrestore(&rcu_get_root(rsp)->lock, flags);
if (needwake) if (needwake)
...@@ -3005,8 +3014,7 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp, ...@@ -3005,8 +3014,7 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
if (!rcu_gp_in_progress(rsp)) { if (!rcu_gp_in_progress(rsp)) {
struct rcu_node *rnp_root = rcu_get_root(rsp); struct rcu_node *rnp_root = rcu_get_root(rsp);
raw_spin_lock(&rnp_root->lock); raw_spin_lock_rcu_node(rnp_root);
smp_mb__after_unlock_lock();
needwake = rcu_start_gp(rsp); needwake = rcu_start_gp(rsp);
raw_spin_unlock(&rnp_root->lock); raw_spin_unlock(&rnp_root->lock);
if (needwake) if (needwake)
...@@ -3365,7 +3373,6 @@ static unsigned long rcu_seq_snap(unsigned long *sp) ...@@ -3365,7 +3373,6 @@ static unsigned long rcu_seq_snap(unsigned long *sp)
{ {
unsigned long s; unsigned long s;
smp_mb(); /* Caller's modifications seen first by other CPUs. */
s = (READ_ONCE(*sp) + 3) & ~0x1; s = (READ_ONCE(*sp) + 3) & ~0x1;
smp_mb(); /* Above access must not bleed into critical section. */ smp_mb(); /* Above access must not bleed into critical section. */
return s; return s;
...@@ -3392,6 +3399,7 @@ static void rcu_exp_gp_seq_end(struct rcu_state *rsp) ...@@ -3392,6 +3399,7 @@ static void rcu_exp_gp_seq_end(struct rcu_state *rsp)
} }
static unsigned long rcu_exp_gp_seq_snap(struct rcu_state *rsp) static unsigned long rcu_exp_gp_seq_snap(struct rcu_state *rsp)
{ {
smp_mb(); /* Caller's modifications seen first by other CPUs. */
return rcu_seq_snap(&rsp->expedited_sequence); return rcu_seq_snap(&rsp->expedited_sequence);
} }
static bool rcu_exp_gp_seq_done(struct rcu_state *rsp, unsigned long s) static bool rcu_exp_gp_seq_done(struct rcu_state *rsp, unsigned long s)
...@@ -3426,8 +3434,7 @@ static void sync_exp_reset_tree_hotplug(struct rcu_state *rsp) ...@@ -3426,8 +3434,7 @@ static void sync_exp_reset_tree_hotplug(struct rcu_state *rsp)
* CPUs for the current rcu_node structure up the rcu_node tree. * CPUs for the current rcu_node structure up the rcu_node tree.
*/ */
rcu_for_each_leaf_node(rsp, rnp) { rcu_for_each_leaf_node(rsp, rnp) {
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
smp_mb__after_unlock_lock();
if (rnp->expmaskinit == rnp->expmaskinitnext) { if (rnp->expmaskinit == rnp->expmaskinitnext) {
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
continue; /* No new CPUs, nothing to do. */ continue; /* No new CPUs, nothing to do. */
...@@ -3447,8 +3454,7 @@ static void sync_exp_reset_tree_hotplug(struct rcu_state *rsp) ...@@ -3447,8 +3454,7 @@ static void sync_exp_reset_tree_hotplug(struct rcu_state *rsp)
rnp_up = rnp->parent; rnp_up = rnp->parent;
done = false; done = false;
while (rnp_up) { while (rnp_up) {
raw_spin_lock_irqsave(&rnp_up->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp_up, flags);
smp_mb__after_unlock_lock();
if (rnp_up->expmaskinit) if (rnp_up->expmaskinit)
done = true; done = true;
rnp_up->expmaskinit |= mask; rnp_up->expmaskinit |= mask;
...@@ -3472,8 +3478,7 @@ static void __maybe_unused sync_exp_reset_tree(struct rcu_state *rsp) ...@@ -3472,8 +3478,7 @@ static void __maybe_unused sync_exp_reset_tree(struct rcu_state *rsp)
sync_exp_reset_tree_hotplug(rsp); sync_exp_reset_tree_hotplug(rsp);
rcu_for_each_node_breadth_first(rsp, rnp) { rcu_for_each_node_breadth_first(rsp, rnp) {
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
smp_mb__after_unlock_lock();
WARN_ON_ONCE(rnp->expmask); WARN_ON_ONCE(rnp->expmask);
rnp->expmask = rnp->expmaskinit; rnp->expmask = rnp->expmaskinit;
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
...@@ -3531,8 +3536,7 @@ static void __rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp, ...@@ -3531,8 +3536,7 @@ static void __rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
mask = rnp->grpmask; mask = rnp->grpmask;
raw_spin_unlock(&rnp->lock); /* irqs remain disabled */ raw_spin_unlock(&rnp->lock); /* irqs remain disabled */
rnp = rnp->parent; rnp = rnp->parent;
raw_spin_lock(&rnp->lock); /* irqs already disabled */ raw_spin_lock_rcu_node(rnp); /* irqs already disabled */
smp_mb__after_unlock_lock();
WARN_ON_ONCE(!(rnp->expmask & mask)); WARN_ON_ONCE(!(rnp->expmask & mask));
rnp->expmask &= ~mask; rnp->expmask &= ~mask;
} }
...@@ -3549,8 +3553,7 @@ static void __maybe_unused rcu_report_exp_rnp(struct rcu_state *rsp, ...@@ -3549,8 +3553,7 @@ static void __maybe_unused rcu_report_exp_rnp(struct rcu_state *rsp,
{ {
unsigned long flags; unsigned long flags;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
smp_mb__after_unlock_lock();
__rcu_report_exp_rnp(rsp, rnp, wake, flags); __rcu_report_exp_rnp(rsp, rnp, wake, flags);
} }
...@@ -3564,8 +3567,7 @@ static void rcu_report_exp_cpu_mult(struct rcu_state *rsp, struct rcu_node *rnp, ...@@ -3564,8 +3567,7 @@ static void rcu_report_exp_cpu_mult(struct rcu_state *rsp, struct rcu_node *rnp,
{ {
unsigned long flags; unsigned long flags;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
smp_mb__after_unlock_lock();
if (!(rnp->expmask & mask)) { if (!(rnp->expmask & mask)) {
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
return; return;
...@@ -3609,7 +3611,7 @@ static bool sync_exp_work_done(struct rcu_state *rsp, struct rcu_node *rnp, ...@@ -3609,7 +3611,7 @@ static bool sync_exp_work_done(struct rcu_state *rsp, struct rcu_node *rnp,
*/ */
static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s) static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s)
{ {
struct rcu_data *rdp; struct rcu_data *rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id());
struct rcu_node *rnp0; struct rcu_node *rnp0;
struct rcu_node *rnp1 = NULL; struct rcu_node *rnp1 = NULL;
...@@ -3623,7 +3625,7 @@ static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s) ...@@ -3623,7 +3625,7 @@ static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s)
if (!mutex_is_locked(&rnp0->exp_funnel_mutex)) { if (!mutex_is_locked(&rnp0->exp_funnel_mutex)) {
if (mutex_trylock(&rnp0->exp_funnel_mutex)) { if (mutex_trylock(&rnp0->exp_funnel_mutex)) {
if (sync_exp_work_done(rsp, rnp0, NULL, if (sync_exp_work_done(rsp, rnp0, NULL,
&rsp->expedited_workdone0, s)) &rdp->expedited_workdone0, s))
return NULL; return NULL;
return rnp0; return rnp0;
} }
...@@ -3637,14 +3639,13 @@ static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s) ...@@ -3637,14 +3639,13 @@ static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s)
* can be inexact, as it is just promoting locality and is not * can be inexact, as it is just promoting locality and is not
* strictly needed for correctness. * strictly needed for correctness.
*/ */
rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id()); if (sync_exp_work_done(rsp, NULL, NULL, &rdp->expedited_workdone1, s))
if (sync_exp_work_done(rsp, NULL, NULL, &rsp->expedited_workdone1, s))
return NULL; return NULL;
mutex_lock(&rdp->exp_funnel_mutex); mutex_lock(&rdp->exp_funnel_mutex);
rnp0 = rdp->mynode; rnp0 = rdp->mynode;
for (; rnp0 != NULL; rnp0 = rnp0->parent) { for (; rnp0 != NULL; rnp0 = rnp0->parent) {
if (sync_exp_work_done(rsp, rnp1, rdp, if (sync_exp_work_done(rsp, rnp1, rdp,
&rsp->expedited_workdone2, s)) &rdp->expedited_workdone2, s))
return NULL; return NULL;
mutex_lock(&rnp0->exp_funnel_mutex); mutex_lock(&rnp0->exp_funnel_mutex);
if (rnp1) if (rnp1)
...@@ -3654,7 +3655,7 @@ static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s) ...@@ -3654,7 +3655,7 @@ static struct rcu_node *exp_funnel_lock(struct rcu_state *rsp, unsigned long s)
rnp1 = rnp0; rnp1 = rnp0;
} }
if (sync_exp_work_done(rsp, rnp1, rdp, if (sync_exp_work_done(rsp, rnp1, rdp,
&rsp->expedited_workdone3, s)) &rdp->expedited_workdone3, s))
return NULL; return NULL;
return rnp1; return rnp1;
} }
...@@ -3708,8 +3709,7 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp, ...@@ -3708,8 +3709,7 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
sync_exp_reset_tree(rsp); sync_exp_reset_tree(rsp);
rcu_for_each_leaf_node(rsp, rnp) { rcu_for_each_leaf_node(rsp, rnp) {
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
smp_mb__after_unlock_lock();
/* Each pass checks a CPU for identity, offline, and idle. */ /* Each pass checks a CPU for identity, offline, and idle. */
mask_ofl_test = 0; mask_ofl_test = 0;
...@@ -3741,25 +3741,23 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp, ...@@ -3741,25 +3741,23 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
ret = smp_call_function_single(cpu, func, rsp, 0); ret = smp_call_function_single(cpu, func, rsp, 0);
if (!ret) { if (!ret) {
mask_ofl_ipi &= ~mask; mask_ofl_ipi &= ~mask;
} else { continue;
}
/* Failed, raced with offline. */ /* Failed, raced with offline. */
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
if (cpu_online(cpu) && if (cpu_online(cpu) &&
(rnp->expmask & mask)) { (rnp->expmask & mask)) {
raw_spin_unlock_irqrestore(&rnp->lock, raw_spin_unlock_irqrestore(&rnp->lock, flags);
flags);
schedule_timeout_uninterruptible(1); schedule_timeout_uninterruptible(1);
if (cpu_online(cpu) && if (cpu_online(cpu) &&
(rnp->expmask & mask)) (rnp->expmask & mask))
goto retry_ipi; goto retry_ipi;
raw_spin_lock_irqsave(&rnp->lock, raw_spin_lock_irqsave_rcu_node(rnp, flags);
flags);
} }
if (!(rnp->expmask & mask)) if (!(rnp->expmask & mask))
mask_ofl_ipi &= ~mask; mask_ofl_ipi &= ~mask;
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
} }
}
/* Report quiescent states for those that went offline. */ /* Report quiescent states for those that went offline. */
mask_ofl_test |= mask_ofl_ipi; mask_ofl_test |= mask_ofl_ipi;
if (mask_ofl_test) if (mask_ofl_test)
...@@ -3773,6 +3771,7 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp) ...@@ -3773,6 +3771,7 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp)
unsigned long jiffies_stall; unsigned long jiffies_stall;
unsigned long jiffies_start; unsigned long jiffies_start;
unsigned long mask; unsigned long mask;
int ndetected;
struct rcu_node *rnp; struct rcu_node *rnp;
struct rcu_node *rnp_root = rcu_get_root(rsp); struct rcu_node *rnp_root = rcu_get_root(rsp);
int ret; int ret;
...@@ -3785,7 +3784,7 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp) ...@@ -3785,7 +3784,7 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp)
rsp->expedited_wq, rsp->expedited_wq,
sync_rcu_preempt_exp_done(rnp_root), sync_rcu_preempt_exp_done(rnp_root),
jiffies_stall); jiffies_stall);
if (ret > 0) if (ret > 0 || sync_rcu_preempt_exp_done(rnp_root))
return; return;
if (ret < 0) { if (ret < 0) {
/* Hit a signal, disable CPU stall warnings. */ /* Hit a signal, disable CPU stall warnings. */
...@@ -3795,14 +3794,16 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp) ...@@ -3795,14 +3794,16 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp)
} }
pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {", pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {",
rsp->name); rsp->name);
ndetected = 0;
rcu_for_each_leaf_node(rsp, rnp) { rcu_for_each_leaf_node(rsp, rnp) {
(void)rcu_print_task_exp_stall(rnp); ndetected = rcu_print_task_exp_stall(rnp);
mask = 1; mask = 1;
for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask <<= 1) { for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask <<= 1) {
struct rcu_data *rdp; struct rcu_data *rdp;
if (!(rnp->expmask & mask)) if (!(rnp->expmask & mask))
continue; continue;
ndetected++;
rdp = per_cpu_ptr(rsp->rda, cpu); rdp = per_cpu_ptr(rsp->rda, cpu);
pr_cont(" %d-%c%c%c", cpu, pr_cont(" %d-%c%c%c", cpu,
"O."[cpu_online(cpu)], "O."[cpu_online(cpu)],
...@@ -3811,8 +3812,23 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp) ...@@ -3811,8 +3812,23 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp)
} }
mask <<= 1; mask <<= 1;
} }
pr_cont(" } %lu jiffies s: %lu\n", pr_cont(" } %lu jiffies s: %lu root: %#lx/%c\n",
jiffies - jiffies_start, rsp->expedited_sequence); jiffies - jiffies_start, rsp->expedited_sequence,
rnp_root->expmask, ".T"[!!rnp_root->exp_tasks]);
if (!ndetected) {
pr_err("blocking rcu_node structures:");
rcu_for_each_node_breadth_first(rsp, rnp) {
if (rnp == rnp_root)
continue; /* printed unconditionally */
if (sync_rcu_preempt_exp_done(rnp))
continue;
pr_cont(" l=%u:%d-%d:%#lx/%c",
rnp->level, rnp->grplo, rnp->grphi,
rnp->expmask,
".T"[!!rnp->exp_tasks]);
}
pr_cont("\n");
}
rcu_for_each_leaf_node(rsp, rnp) { rcu_for_each_leaf_node(rsp, rnp) {
mask = 1; mask = 1;
for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask <<= 1) { for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask <<= 1) {
...@@ -3847,6 +3863,16 @@ void synchronize_sched_expedited(void) ...@@ -3847,6 +3863,16 @@ void synchronize_sched_expedited(void)
struct rcu_node *rnp; struct rcu_node *rnp;
struct rcu_state *rsp = &rcu_sched_state; struct rcu_state *rsp = &rcu_sched_state;
/* If only one CPU, this is automatically a grace period. */
if (rcu_blocking_is_gp())
return;
/* If expedited grace periods are prohibited, fall back to normal. */
if (rcu_gp_is_normal()) {
wait_rcu_gp(call_rcu_sched);
return;
}
/* Take a snapshot of the sequence number. */ /* Take a snapshot of the sequence number. */
s = rcu_exp_gp_seq_snap(rsp); s = rcu_exp_gp_seq_snap(rsp);
...@@ -4135,7 +4161,7 @@ static void rcu_init_new_rnp(struct rcu_node *rnp_leaf) ...@@ -4135,7 +4161,7 @@ static void rcu_init_new_rnp(struct rcu_node *rnp_leaf)
rnp = rnp->parent; rnp = rnp->parent;
if (rnp == NULL) if (rnp == NULL)
return; return;
raw_spin_lock(&rnp->lock); /* Interrupts already disabled. */ raw_spin_lock_rcu_node(rnp); /* Interrupts already disabled. */
rnp->qsmaskinit |= mask; rnp->qsmaskinit |= mask;
raw_spin_unlock(&rnp->lock); /* Interrupts remain disabled. */ raw_spin_unlock(&rnp->lock); /* Interrupts remain disabled. */
} }
...@@ -4152,7 +4178,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp) ...@@ -4152,7 +4178,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
/* Set up local state, ensuring consistent view of global state. */ /* Set up local state, ensuring consistent view of global state. */
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo); rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
rdp->dynticks = &per_cpu(rcu_dynticks, cpu); rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE); WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE);
...@@ -4179,7 +4205,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp) ...@@ -4179,7 +4205,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
/* Set up local state, ensuring consistent view of global state. */ /* Set up local state, ensuring consistent view of global state. */
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
rdp->qlen_last_fqs_check = 0; rdp->qlen_last_fqs_check = 0;
rdp->n_force_qs_snap = rsp->n_force_qs; rdp->n_force_qs_snap = rsp->n_force_qs;
rdp->blimit = blimit; rdp->blimit = blimit;
...@@ -4198,8 +4224,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp) ...@@ -4198,8 +4224,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
*/ */
rnp = rdp->mynode; rnp = rdp->mynode;
mask = rdp->grpmask; mask = rdp->grpmask;
raw_spin_lock(&rnp->lock); /* irqs already disabled. */ raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
smp_mb__after_unlock_lock();
rnp->qsmaskinitnext |= mask; rnp->qsmaskinitnext |= mask;
rnp->expmaskinitnext |= mask; rnp->expmaskinitnext |= mask;
if (!rdp->beenonline) if (!rdp->beenonline)
...@@ -4327,14 +4352,14 @@ static int __init rcu_spawn_gp_kthread(void) ...@@ -4327,14 +4352,14 @@ static int __init rcu_spawn_gp_kthread(void)
t = kthread_create(rcu_gp_kthread, rsp, "%s", rsp->name); t = kthread_create(rcu_gp_kthread, rsp, "%s", rsp->name);
BUG_ON(IS_ERR(t)); BUG_ON(IS_ERR(t));
rnp = rcu_get_root(rsp); rnp = rcu_get_root(rsp);
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
rsp->gp_kthread = t; rsp->gp_kthread = t;
if (kthread_prio) { if (kthread_prio) {
sp.sched_priority = kthread_prio; sp.sched_priority = kthread_prio;
sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
} }
wake_up_process(t);
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
wake_up_process(t);
} }
rcu_spawn_nocb_kthreads(); rcu_spawn_nocb_kthreads();
rcu_spawn_boost_kthreads(); rcu_spawn_boost_kthreads();
...@@ -4385,12 +4410,14 @@ static void __init rcu_init_levelspread(int *levelspread, const int *levelcnt) ...@@ -4385,12 +4410,14 @@ static void __init rcu_init_levelspread(int *levelspread, const int *levelcnt)
/* /*
* Helper function for rcu_init() that initializes one rcu_state structure. * Helper function for rcu_init() that initializes one rcu_state structure.
*/ */
static void __init rcu_init_one(struct rcu_state *rsp, static void __init rcu_init_one(struct rcu_state *rsp)
struct rcu_data __percpu *rda)
{ {
static const char * const buf[] = RCU_NODE_NAME_INIT; static const char * const buf[] = RCU_NODE_NAME_INIT;
static const char * const fqs[] = RCU_FQS_NAME_INIT; static const char * const fqs[] = RCU_FQS_NAME_INIT;
static const char * const exp[] = RCU_EXP_NAME_INIT; static const char * const exp[] = RCU_EXP_NAME_INIT;
static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
static struct lock_class_key rcu_exp_class[RCU_NUM_LVLS];
static u8 fl_mask = 0x1; static u8 fl_mask = 0x1;
int levelcnt[RCU_NUM_LVLS]; /* # nodes in each level. */ int levelcnt[RCU_NUM_LVLS]; /* # nodes in each level. */
...@@ -4576,8 +4603,8 @@ void __init rcu_init(void) ...@@ -4576,8 +4603,8 @@ void __init rcu_init(void)
rcu_bootup_announce(); rcu_bootup_announce();
rcu_init_geometry(); rcu_init_geometry();
rcu_init_one(&rcu_bh_state, &rcu_bh_data); rcu_init_one(&rcu_bh_state);
rcu_init_one(&rcu_sched_state, &rcu_sched_data); rcu_init_one(&rcu_sched_state);
if (dump_tree) if (dump_tree)
rcu_dump_rcu_node_tree(&rcu_sched_state); rcu_dump_rcu_node_tree(&rcu_sched_state);
__rcu_init_preempt(); __rcu_init_preempt();
......
kernel/rcu/tree.h
View file @ 3104fb3d
...@@ -178,6 +178,8 @@ struct rcu_node { ...@@ -178,6 +178,8 @@ struct rcu_node {
/* beginning of each expedited GP. */ /* beginning of each expedited GP. */
unsigned long expmaskinitnext; unsigned long expmaskinitnext;
/* Online CPUs for next expedited GP. */ /* Online CPUs for next expedited GP. */
/* Any CPU that has ever been online will */
/* have its bit set. */
unsigned long grpmask; /* Mask to apply to parent qsmask. */ unsigned long grpmask; /* Mask to apply to parent qsmask. */
/* Only one bit will be set in this mask. */ /* Only one bit will be set in this mask. */
int grplo; /* lowest-numbered CPU or group here. */ int grplo; /* lowest-numbered CPU or group here. */
...@@ -384,6 +386,10 @@ struct rcu_data { ...@@ -384,6 +386,10 @@ struct rcu_data {
struct rcu_head oom_head; struct rcu_head oom_head;
#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
struct mutex exp_funnel_mutex; struct mutex exp_funnel_mutex;
atomic_long_t expedited_workdone0; /* # done by others #0. */
atomic_long_t expedited_workdone1; /* # done by others #1. */
atomic_long_t expedited_workdone2; /* # done by others #2. */
atomic_long_t expedited_workdone3; /* # done by others #3. */
/* 7) Callback offloading. */ /* 7) Callback offloading. */
#ifdef CONFIG_RCU_NOCB_CPU #ifdef CONFIG_RCU_NOCB_CPU
...@@ -498,10 +504,6 @@ struct rcu_state { ...@@ -498,10 +504,6 @@ struct rcu_state {
/* End of fields guarded by barrier_mutex. */ /* End of fields guarded by barrier_mutex. */
unsigned long expedited_sequence; /* Take a ticket. */ unsigned long expedited_sequence; /* Take a ticket. */
atomic_long_t expedited_workdone0; /* # done by others #0. */
atomic_long_t expedited_workdone1; /* # done by others #1. */
atomic_long_t expedited_workdone2; /* # done by others #2. */
atomic_long_t expedited_workdone3; /* # done by others #3. */
atomic_long_t expedited_normal; /* # fallbacks to normal. */ atomic_long_t expedited_normal; /* # fallbacks to normal. */
atomic_t expedited_need_qs; /* # CPUs left to check in. */ atomic_t expedited_need_qs; /* # CPUs left to check in. */
wait_queue_head_t expedited_wq; /* Wait for check-ins. */ wait_queue_head_t expedited_wq; /* Wait for check-ins. */
...@@ -545,6 +547,18 @@ struct rcu_state { ...@@ -545,6 +547,18 @@ struct rcu_state {
#define RCU_GP_CLEANUP 5 /* Grace-period cleanup started. */ #define RCU_GP_CLEANUP 5 /* Grace-period cleanup started. */
#define RCU_GP_CLEANED 6 /* Grace-period cleanup complete. */ #define RCU_GP_CLEANED 6 /* Grace-period cleanup complete. */
#ifndef RCU_TREE_NONCORE
static const char * const gp_state_names[] = {
"RCU_GP_IDLE",
"RCU_GP_WAIT_GPS",
"RCU_GP_DONE_GPS",
"RCU_GP_WAIT_FQS",
"RCU_GP_DOING_FQS",
"RCU_GP_CLEANUP",
"RCU_GP_CLEANED",
};
#endif /* #ifndef RCU_TREE_NONCORE */
extern struct list_head rcu_struct_flavors; extern struct list_head rcu_struct_flavors;
/* Sequence through rcu_state structures for each RCU flavor. */ /* Sequence through rcu_state structures for each RCU flavor. */
...@@ -664,3 +678,42 @@ static inline void rcu_nocb_q_lengths(struct rcu_data *rdp, long *ql, long *qll) ...@@ -664,3 +678,42 @@ static inline void rcu_nocb_q_lengths(struct rcu_data *rdp, long *ql, long *qll)
#else /* #ifdef CONFIG_PPC */ #else /* #ifdef CONFIG_PPC */
#define smp_mb__after_unlock_lock() do { } while (0) #define smp_mb__after_unlock_lock() do { } while (0)
#endif /* #else #ifdef CONFIG_PPC */ #endif /* #else #ifdef CONFIG_PPC */
/*
* Wrappers for the rcu_node::lock acquire.
*
* Because the rcu_nodes form a tree, the tree traversal locking will observe
* different lock values, this in turn means that an UNLOCK of one level
* followed by a LOCK of another level does not imply a full memory barrier;
* and most importantly transitivity is lost.
*
* In order to restore full ordering between tree levels, augment the regular
* lock acquire functions with smp_mb__after_unlock_lock().
*/
static inline void raw_spin_lock_rcu_node(struct rcu_node *rnp)
{
raw_spin_lock(&rnp->lock);
smp_mb__after_unlock_lock();
}
static inline void raw_spin_lock_irq_rcu_node(struct rcu_node *rnp)
{
raw_spin_lock_irq(&rnp->lock);
smp_mb__after_unlock_lock();
}
#define raw_spin_lock_irqsave_rcu_node(rnp, flags) \
do { \
typecheck(unsigned long, flags); \
raw_spin_lock_irqsave(&(rnp)->lock, flags); \
smp_mb__after_unlock_lock(); \
} while (0)
static inline bool raw_spin_trylock_rcu_node(struct rcu_node *rnp)
{
bool locked = raw_spin_trylock(&rnp->lock);
if (locked)
smp_mb__after_unlock_lock();
return locked;
}
kernel/rcu/tree_plugin.h
View file @ 3104fb3d
...@@ -63,8 +63,7 @@ static bool __read_mostly rcu_nocb_poll; /* Offload kthread are to poll. */ ...@@ -63,8 +63,7 @@ static bool __read_mostly rcu_nocb_poll; /* Offload kthread are to poll. */
/* /*
* Check the RCU kernel configuration parameters and print informative * Check the RCU kernel configuration parameters and print informative
* messages about anything out of the ordinary. If you like #ifdef, you * messages about anything out of the ordinary.
* will love this function.
*/ */
static void __init rcu_bootup_announce_oddness(void) static void __init rcu_bootup_announce_oddness(void)
{ {
...@@ -147,8 +146,8 @@ static void __init rcu_bootup_announce(void) ...@@ -147,8 +146,8 @@ static void __init rcu_bootup_announce(void)
* the corresponding expedited grace period will also be the end of the * the corresponding expedited grace period will also be the end of the
* normal grace period. * normal grace period.
*/ */
static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp, static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)
unsigned long flags) __releases(rnp->lock) __releases(rnp->lock) /* But leaves rrupts disabled. */
{ {
int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) + int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) +
(rnp->exp_tasks ? RCU_EXP_TASKS : 0) + (rnp->exp_tasks ? RCU_EXP_TASKS : 0) +
...@@ -236,7 +235,7 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp, ...@@ -236,7 +235,7 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp,
rnp->gp_tasks = &t->rcu_node_entry; rnp->gp_tasks = &t->rcu_node_entry;
if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD)) if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD))
rnp->exp_tasks = &t->rcu_node_entry; rnp->exp_tasks = &t->rcu_node_entry;
raw_spin_unlock(&rnp->lock); raw_spin_unlock(&rnp->lock); /* rrupts remain disabled. */
/* /*
* Report the quiescent state for the expedited GP. This expedited * Report the quiescent state for the expedited GP. This expedited
...@@ -251,7 +250,6 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp, ...@@ -251,7 +250,6 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp,
} else { } else {
WARN_ON_ONCE(t->rcu_read_unlock_special.b.exp_need_qs); WARN_ON_ONCE(t->rcu_read_unlock_special.b.exp_need_qs);
} }
local_irq_restore(flags);
} }
/* /*
...@@ -286,12 +284,11 @@ static void rcu_preempt_qs(void) ...@@ -286,12 +284,11 @@ static void rcu_preempt_qs(void)
* predating the current grace period drain, in other words, until * predating the current grace period drain, in other words, until
* rnp->gp_tasks becomes NULL. * rnp->gp_tasks becomes NULL.
* *
* Caller must disable preemption. * Caller must disable interrupts.
*/ */
static void rcu_preempt_note_context_switch(void) static void rcu_preempt_note_context_switch(void)
{ {
struct task_struct *t = current; struct task_struct *t = current;
unsigned long flags;
struct rcu_data *rdp; struct rcu_data *rdp;
struct rcu_node *rnp; struct rcu_node *rnp;
...@@ -301,8 +298,7 @@ static void rcu_preempt_note_context_switch(void) ...@@ -301,8 +298,7 @@ static void rcu_preempt_note_context_switch(void)
/* Possibly blocking in an RCU read-side critical section. */ /* Possibly blocking in an RCU read-side critical section. */
rdp = this_cpu_ptr(rcu_state_p->rda); rdp = this_cpu_ptr(rcu_state_p->rda);
rnp = rdp->mynode; rnp = rdp->mynode;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_rcu_node(rnp);
smp_mb__after_unlock_lock();
t->rcu_read_unlock_special.b.blocked = true; t->rcu_read_unlock_special.b.blocked = true;
t->rcu_blocked_node = rnp; t->rcu_blocked_node = rnp;
...@@ -318,7 +314,7 @@ static void rcu_preempt_note_context_switch(void) ...@@ -318,7 +314,7 @@ static void rcu_preempt_note_context_switch(void)
(rnp->qsmask & rdp->grpmask) (rnp->qsmask & rdp->grpmask)
? rnp->gpnum ? rnp->gpnum
: rnp->gpnum + 1); : rnp->gpnum + 1);
rcu_preempt_ctxt_queue(rnp, rdp, flags); rcu_preempt_ctxt_queue(rnp, rdp);
} else if (t->rcu_read_lock_nesting < 0 && } else if (t->rcu_read_lock_nesting < 0 &&
t->rcu_read_unlock_special.s) { t->rcu_read_unlock_special.s) {
...@@ -450,20 +446,13 @@ void rcu_read_unlock_special(struct task_struct *t) ...@@ -450,20 +446,13 @@ void rcu_read_unlock_special(struct task_struct *t)
/* /*
* Remove this task from the list it blocked on. The task * Remove this task from the list it blocked on. The task
* now remains queued on the rcu_node corresponding to * now remains queued on the rcu_node corresponding to the
* the CPU it first blocked on, so the first attempt to * CPU it first blocked on, so there is no longer any need
* acquire the task's rcu_node's ->lock will succeed. * to loop. Retain a WARN_ON_ONCE() out of sheer paranoia.
* Keep the loop and add a WARN_ON() out of sheer paranoia.
*/ */
for (;;) {
rnp = t->rcu_blocked_node; rnp = t->rcu_blocked_node;
raw_spin_lock(&rnp->lock); /* irqs already disabled. */ raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
smp_mb__after_unlock_lock(); WARN_ON_ONCE(rnp != t->rcu_blocked_node);
if (rnp == t->rcu_blocked_node)
break;
WARN_ON_ONCE(1);
raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
}
empty_norm = !rcu_preempt_blocked_readers_cgp(rnp); empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
empty_exp = sync_rcu_preempt_exp_done(rnp); empty_exp = sync_rcu_preempt_exp_done(rnp);
smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */ smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
...@@ -527,7 +516,7 @@ static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp) ...@@ -527,7 +516,7 @@ static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
unsigned long flags; unsigned long flags;
struct task_struct *t; struct task_struct *t;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
if (!rcu_preempt_blocked_readers_cgp(rnp)) { if (!rcu_preempt_blocked_readers_cgp(rnp)) {
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
return; return;
...@@ -748,6 +737,12 @@ void synchronize_rcu_expedited(void) ...@@ -748,6 +737,12 @@ void synchronize_rcu_expedited(void)
struct rcu_state *rsp = rcu_state_p; struct rcu_state *rsp = rcu_state_p;
unsigned long s; unsigned long s;
/* If expedited grace periods are prohibited, fall back to normal. */
if (rcu_gp_is_normal()) {
wait_rcu_gp(call_rcu);
return;
}
s = rcu_exp_gp_seq_snap(rsp); s = rcu_exp_gp_seq_snap(rsp);
rnp_unlock = exp_funnel_lock(rsp, s); rnp_unlock = exp_funnel_lock(rsp, s);
...@@ -788,7 +783,7 @@ EXPORT_SYMBOL_GPL(rcu_barrier); ...@@ -788,7 +783,7 @@ EXPORT_SYMBOL_GPL(rcu_barrier);
*/ */
static void __init __rcu_init_preempt(void) static void __init __rcu_init_preempt(void)
{ {
rcu_init_one(rcu_state_p, rcu_data_p); rcu_init_one(rcu_state_p);
} }
/* /*
...@@ -989,8 +984,7 @@ static int rcu_boost(struct rcu_node *rnp) ...@@ -989,8 +984,7 @@ static int rcu_boost(struct rcu_node *rnp)
READ_ONCE(rnp->boost_tasks) == NULL) READ_ONCE(rnp->boost_tasks) == NULL)
return 0; /* Nothing left to boost. */ return 0; /* Nothing left to boost. */
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
smp_mb__after_unlock_lock();
/* /*
* Recheck under the lock: all tasks in need of boosting * Recheck under the lock: all tasks in need of boosting
...@@ -1176,8 +1170,7 @@ static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, ...@@ -1176,8 +1170,7 @@ static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
"rcub/%d", rnp_index); "rcub/%d", rnp_index);
if (IS_ERR(t)) if (IS_ERR(t))
return PTR_ERR(t); return PTR_ERR(t);
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
smp_mb__after_unlock_lock();
rnp->boost_kthread_task = t; rnp->boost_kthread_task = t;
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
sp.sched_priority = kthread_prio; sp.sched_priority = kthread_prio;
...@@ -1524,7 +1517,8 @@ static void rcu_prepare_for_idle(void) ...@@ -1524,7 +1517,8 @@ static void rcu_prepare_for_idle(void)
struct rcu_state *rsp; struct rcu_state *rsp;
int tne; int tne;
if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)) if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) ||
rcu_is_nocb_cpu(smp_processor_id()))
return; return;
/* Handle nohz enablement switches conservatively. */ /* Handle nohz enablement switches conservatively. */
...@@ -1538,10 +1532,6 @@ static void rcu_prepare_for_idle(void) ...@@ -1538,10 +1532,6 @@ static void rcu_prepare_for_idle(void)
if (!tne) if (!tne)
return; return;
/* If this is a no-CBs CPU, no callbacks, just return. */
if (rcu_is_nocb_cpu(smp_processor_id()))
return;
/* /*
* If a non-lazy callback arrived at a CPU having only lazy * If a non-lazy callback arrived at a CPU having only lazy
* callbacks, invoke RCU core for the side-effect of recalculating * callbacks, invoke RCU core for the side-effect of recalculating
...@@ -1567,8 +1557,7 @@ static void rcu_prepare_for_idle(void) ...@@ -1567,8 +1557,7 @@ static void rcu_prepare_for_idle(void)
if (!*rdp->nxttail[RCU_DONE_TAIL]) if (!*rdp->nxttail[RCU_DONE_TAIL])
continue; continue;
rnp = rdp->mynode; rnp = rdp->mynode;
raw_spin_lock(&rnp->lock); /* irqs already disabled. */ raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
smp_mb__after_unlock_lock();
needwake = rcu_accelerate_cbs(rsp, rnp, rdp); needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */ raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
if (needwake) if (needwake)
...@@ -2068,8 +2057,7 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp) ...@@ -2068,8 +2057,7 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
bool needwake; bool needwake;
struct rcu_node *rnp = rdp->mynode; struct rcu_node *rnp = rdp->mynode;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
smp_mb__after_unlock_lock();
needwake = rcu_start_future_gp(rnp, rdp, &c); needwake = rcu_start_future_gp(rnp, rdp, &c);
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
if (needwake) if (needwake)
......
kernel/rcu/tree_trace.c
View file @ 3104fb3d
/* /*
* Read-Copy Update tracing for classic implementation * Read-Copy Update tracing for hierarchical implementation.
* *
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
* http://www.gnu.org/licenses/gpl-2.0.html. * http://www.gnu.org/licenses/gpl-2.0.html.
* *
* Copyright IBM Corporation, 2008 * Copyright IBM Corporation, 2008
* Author: Paul E. McKenney
* *
* Papers: http://www.rdrop.com/users/paulmck/RCU * Papers: http://www.rdrop.com/users/paulmck/RCU
* *
...@@ -33,9 +34,7 @@ ...@@ -33,9 +34,7 @@
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/atomic.h> #include <linux/atomic.h>
#include <linux/bitops.h> #include <linux/bitops.h>
#include <linux/module.h>
#include <linux/completion.h> #include <linux/completion.h>
#include <linux/moduleparam.h>
#include <linux/percpu.h> #include <linux/percpu.h>
#include <linux/notifier.h> #include <linux/notifier.h>
#include <linux/cpu.h> #include <linux/cpu.h>
...@@ -183,14 +182,20 @@ static const struct file_operations rcudata_fops = { ...@@ -183,14 +182,20 @@ static const struct file_operations rcudata_fops = {
static int show_rcuexp(struct seq_file *m, void *v) static int show_rcuexp(struct seq_file *m, void *v)
{ {
int cpu;
struct rcu_state *rsp = (struct rcu_state *)m->private; struct rcu_state *rsp = (struct rcu_state *)m->private;
struct rcu_data *rdp;
unsigned long s0 = 0, s1 = 0, s2 = 0, s3 = 0;
for_each_possible_cpu(cpu) {
rdp = per_cpu_ptr(rsp->rda, cpu);
s0 += atomic_long_read(&rdp->expedited_workdone0);
s1 += atomic_long_read(&rdp->expedited_workdone1);
s2 += atomic_long_read(&rdp->expedited_workdone2);
s3 += atomic_long_read(&rdp->expedited_workdone3);
}
seq_printf(m, "s=%lu wd0=%lu wd1=%lu wd2=%lu wd3=%lu n=%lu enq=%d sc=%lu\n", seq_printf(m, "s=%lu wd0=%lu wd1=%lu wd2=%lu wd3=%lu n=%lu enq=%d sc=%lu\n",
rsp->expedited_sequence, rsp->expedited_sequence, s0, s1, s2, s3,
atomic_long_read(&rsp->expedited_workdone0),
atomic_long_read(&rsp->expedited_workdone1),
atomic_long_read(&rsp->expedited_workdone2),
atomic_long_read(&rsp->expedited_workdone3),
atomic_long_read(&rsp->expedited_normal), atomic_long_read(&rsp->expedited_normal),
atomic_read(&rsp->expedited_need_qs), atomic_read(&rsp->expedited_need_qs),
rsp->expedited_sequence / 2); rsp->expedited_sequence / 2);
...@@ -319,7 +324,7 @@ static void show_one_rcugp(struct seq_file *m, struct rcu_state *rsp) ...@@ -319,7 +324,7 @@ static void show_one_rcugp(struct seq_file *m, struct rcu_state *rsp)
unsigned long gpmax; unsigned long gpmax;
struct rcu_node *rnp = &rsp->node[0]; struct rcu_node *rnp = &rsp->node[0];
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
completed = READ_ONCE(rsp->completed); completed = READ_ONCE(rsp->completed);
gpnum = READ_ONCE(rsp->gpnum); gpnum = READ_ONCE(rsp->gpnum);
if (completed == gpnum) if (completed == gpnum)
...@@ -487,16 +492,4 @@ static int __init rcutree_trace_init(void) ...@@ -487,16 +492,4 @@ static int __init rcutree_trace_init(void)
debugfs_remove_recursive(rcudir); debugfs_remove_recursive(rcudir);
return 1; return 1;
} }
device_initcall(rcutree_trace_init);
static void __exit rcutree_trace_cleanup(void)
{
debugfs_remove_recursive(rcudir);
}
module_init(rcutree_trace_init);
module_exit(rcutree_trace_cleanup);
MODULE_AUTHOR("Paul E. McKenney");
MODULE_DESCRIPTION("Read-Copy Update tracing for hierarchical implementation");
MODULE_LICENSE("GPL");
kernel/rcu/update.c
View file @ 3104fb3d
...@@ -60,7 +60,12 @@ MODULE_ALIAS("rcupdate"); ...@@ -60,7 +60,12 @@ MODULE_ALIAS("rcupdate");
#endif #endif
#define MODULE_PARAM_PREFIX "rcupdate." #define MODULE_PARAM_PREFIX "rcupdate."
#ifndef CONFIG_TINY_RCU
module_param(rcu_expedited, int, 0); module_param(rcu_expedited, int, 0);
module_param(rcu_normal, int, 0);
static int rcu_normal_after_boot;
module_param(rcu_normal_after_boot, int, 0);
#endif /* #ifndef CONFIG_TINY_RCU */
#if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_PREEMPT_COUNT) #if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_PREEMPT_COUNT)
/** /**
...@@ -113,6 +118,17 @@ EXPORT_SYMBOL(rcu_read_lock_sched_held); ...@@ -113,6 +118,17 @@ EXPORT_SYMBOL(rcu_read_lock_sched_held);
#ifndef CONFIG_TINY_RCU #ifndef CONFIG_TINY_RCU
/*
* Should expedited grace-period primitives always fall back to their
* non-expedited counterparts? Intended for use within RCU. Note
* that if the user specifies both rcu_expedited and rcu_normal, then
* rcu_normal wins.
*/
bool rcu_gp_is_normal(void)
{
return READ_ONCE(rcu_normal);
}
static atomic_t rcu_expedited_nesting = static atomic_t rcu_expedited_nesting =
ATOMIC_INIT(IS_ENABLED(CONFIG_RCU_EXPEDITE_BOOT) ? 1 : 0); ATOMIC_INIT(IS_ENABLED(CONFIG_RCU_EXPEDITE_BOOT) ? 1 : 0);
...@@ -157,8 +173,6 @@ void rcu_unexpedite_gp(void) ...@@ -157,8 +173,6 @@ void rcu_unexpedite_gp(void)
} }
EXPORT_SYMBOL_GPL(rcu_unexpedite_gp); EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);
#endif /* #ifndef CONFIG_TINY_RCU */
/* /*
* Inform RCU of the end of the in-kernel boot sequence. * Inform RCU of the end of the in-kernel boot sequence.
*/ */
...@@ -166,8 +180,12 @@ void rcu_end_inkernel_boot(void) ...@@ -166,8 +180,12 @@ void rcu_end_inkernel_boot(void)
{ {
if (IS_ENABLED(CONFIG_RCU_EXPEDITE_BOOT)) if (IS_ENABLED(CONFIG_RCU_EXPEDITE_BOOT))
rcu_unexpedite_gp(); rcu_unexpedite_gp();
if (rcu_normal_after_boot)
WRITE_ONCE(rcu_normal, 1);
} }
#endif /* #ifndef CONFIG_TINY_RCU */
#ifdef CONFIG_PREEMPT_RCU #ifdef CONFIG_PREEMPT_RCU
/* /*
......
kernel/sched/core.c
View file @ 3104fb3d
...@@ -3109,7 +3109,6 @@ static void __sched notrace __schedule(bool preempt) ...@@ -3109,7 +3109,6 @@ static void __sched notrace __schedule(bool preempt)
cpu = smp_processor_id(); cpu = smp_processor_id();
rq = cpu_rq(cpu); rq = cpu_rq(cpu);
rcu_note_context_switch();
prev = rq->curr; prev = rq->curr;
/* /*
...@@ -3128,13 +3127,16 @@ static void __sched notrace __schedule(bool preempt) ...@@ -3128,13 +3127,16 @@ static void __sched notrace __schedule(bool preempt)
if (sched_feat(HRTICK)) if (sched_feat(HRTICK))
hrtick_clear(rq); hrtick_clear(rq);
local_irq_disable();
rcu_note_context_switch();
/* /*
* Make sure that signal_pending_state()->signal_pending() below * Make sure that signal_pending_state()->signal_pending() below
* can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
* done by the caller to avoid the race with signal_wake_up(). * done by the caller to avoid the race with signal_wake_up().
*/ */
smp_mb__before_spinlock(); smp_mb__before_spinlock();
raw_spin_lock_irq(&rq->lock); raw_spin_lock(&rq->lock);
lockdep_pin_lock(&rq->lock); lockdep_pin_lock(&rq->lock);
rq->clock_skip_update <<= 1; /* promote REQ to ACT */ rq->clock_skip_update <<= 1; /* promote REQ to ACT */
......
lib/list_debug.c
View file @ 3104fb3d
...@@ -37,7 +37,7 @@ void __list_add(struct list_head *new, ...@@ -37,7 +37,7 @@ void __list_add(struct list_head *new,
next->prev = new; next->prev = new;
new->next = next; new->next = next;
new->prev = prev; new->prev = prev;
prev->next = new; WRITE_ONCE(prev->next, new);
} }
EXPORT_SYMBOL(__list_add); EXPORT_SYMBOL(__list_add);
......
tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh
View file @ 3104fb3d
...@@ -38,8 +38,6 @@ ...@@ -38,8 +38,6 @@
# #
# Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com> # Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
grace=120
T=/tmp/kvm-test-1-run.sh.$$ T=/tmp/kvm-test-1-run.sh.$$
trap 'rm -rf $T' 0 trap 'rm -rf $T' 0
touch $T touch $T
...@@ -152,7 +150,7 @@ fi ...@@ -152,7 +150,7 @@ fi
qemu_args="`specify_qemu_cpus "$QEMU" "$qemu_args" "$cpu_count"`" qemu_args="`specify_qemu_cpus "$QEMU" "$qemu_args" "$cpu_count"`"
# Generate architecture-specific and interaction-specific qemu arguments # Generate architecture-specific and interaction-specific qemu arguments
qemu_args="$qemu_args `identify_qemu_args "$QEMU" "$builddir/console.log"`" qemu_args="$qemu_args `identify_qemu_args "$QEMU" "$resdir/console.log"`"
# Generate qemu -append arguments # Generate qemu -append arguments
qemu_append="`identify_qemu_append "$QEMU"`" qemu_append="`identify_qemu_append "$QEMU"`"
...@@ -168,7 +166,7 @@ then ...@@ -168,7 +166,7 @@ then
touch $resdir/buildonly touch $resdir/buildonly
exit 0 exit 0
fi fi
echo "NOTE: $QEMU either did not run or was interactive" > $builddir/console.log echo "NOTE: $QEMU either did not run or was interactive" > $resdir/console.log
echo $QEMU $qemu_args -m 512 -kernel $resdir/bzImage -append \"$qemu_append $boot_args\" > $resdir/qemu-cmd echo $QEMU $qemu_args -m 512 -kernel $resdir/bzImage -append \"$qemu_append $boot_args\" > $resdir/qemu-cmd
( $QEMU $qemu_args -m 512 -kernel $resdir/bzImage -append "$qemu_append $boot_args"; echo $? > $resdir/qemu-retval ) & ( $QEMU $qemu_args -m 512 -kernel $resdir/bzImage -append "$qemu_append $boot_args"; echo $? > $resdir/qemu-retval ) &
qemu_pid=$! qemu_pid=$!
...@@ -214,7 +212,7 @@ then ...@@ -214,7 +212,7 @@ then
else else
break break
fi fi
if test $kruntime -ge $((seconds + grace)) if test $kruntime -ge $((seconds + $TORTURE_SHUTDOWN_GRACE))
then then
echo "!!! PID $qemu_pid hung at $kruntime vs. $seconds seconds" >> $resdir/Warnings 2>&1 echo "!!! PID $qemu_pid hung at $kruntime vs. $seconds seconds" >> $resdir/Warnings 2>&1
kill -KILL $qemu_pid kill -KILL $qemu_pid
...@@ -224,6 +222,5 @@ then ...@@ -224,6 +222,5 @@ then
done done
fi fi
cp $builddir/console.log $resdir
parse-torture.sh $resdir/console.log $title parse-torture.sh $resdir/console.log $title
parse-console.sh $resdir/console.log $title parse-console.sh $resdir/console.log $title
tools/testing/selftests/rcutorture/bin/kvm.sh
View file @ 3104fb3d
...@@ -42,6 +42,7 @@ TORTURE_DEFCONFIG=defconfig ...@@ -42,6 +42,7 @@ TORTURE_DEFCONFIG=defconfig
TORTURE_BOOT_IMAGE="" TORTURE_BOOT_IMAGE=""
TORTURE_INITRD="$KVM/initrd"; export TORTURE_INITRD TORTURE_INITRD="$KVM/initrd"; export TORTURE_INITRD
TORTURE_KMAKE_ARG="" TORTURE_KMAKE_ARG=""
TORTURE_SHUTDOWN_GRACE=180
TORTURE_SUITE=rcu TORTURE_SUITE=rcu
resdir="" resdir=""
configs="" configs=""
...@@ -149,6 +150,11 @@ do ...@@ -149,6 +150,11 @@ do
resdir=$2 resdir=$2
shift shift
;; ;;
--shutdown-grace)
checkarg --shutdown-grace "(seconds)" "$#" "$2" '^[0-9]*$' '^error'
TORTURE_SHUTDOWN_GRACE=$2
shift
;;
--torture) --torture)
checkarg --torture "(suite name)" "$#" "$2" '^\(lock\|rcu\)$' '^--' checkarg --torture "(suite name)" "$#" "$2" '^\(lock\|rcu\)$' '^--'
TORTURE_SUITE=$2 TORTURE_SUITE=$2
...@@ -266,6 +272,7 @@ TORTURE_KMAKE_ARG="$TORTURE_KMAKE_ARG"; export TORTURE_KMAKE_ARG ...@@ -266,6 +272,7 @@ TORTURE_KMAKE_ARG="$TORTURE_KMAKE_ARG"; export TORTURE_KMAKE_ARG
TORTURE_QEMU_CMD="$TORTURE_QEMU_CMD"; export TORTURE_QEMU_CMD TORTURE_QEMU_CMD="$TORTURE_QEMU_CMD"; export TORTURE_QEMU_CMD
TORTURE_QEMU_INTERACTIVE="$TORTURE_QEMU_INTERACTIVE"; export TORTURE_QEMU_INTERACTIVE TORTURE_QEMU_INTERACTIVE="$TORTURE_QEMU_INTERACTIVE"; export TORTURE_QEMU_INTERACTIVE
TORTURE_QEMU_MAC="$TORTURE_QEMU_MAC"; export TORTURE_QEMU_MAC TORTURE_QEMU_MAC="$TORTURE_QEMU_MAC"; export TORTURE_QEMU_MAC
TORTURE_SHUTDOWN_GRACE="$TORTURE_SHUTDOWN_GRACE"; export TORTURE_SHUTDOWN_GRACE
TORTURE_SUITE="$TORTURE_SUITE"; export TORTURE_SUITE TORTURE_SUITE="$TORTURE_SUITE"; export TORTURE_SUITE
if ! test -e $resdir if ! test -e $resdir
then then
...@@ -307,10 +314,10 @@ awk < $T/cfgcpu.pack \ ...@@ -307,10 +314,10 @@ awk < $T/cfgcpu.pack \
} }
# Dump out the scripting required to run one test batch. # Dump out the scripting required to run one test batch.
function dump(first, pastlast) function dump(first, pastlast, batchnum)
{ {
print "echo ----Start batch: `date`"; print "echo ----Start batch " batchnum ": `date`";
print "echo ----Start batch: `date` >> " rd "/log"; print "echo ----Start batch " batchnum ": `date` >> " rd "/log";
jn=1 jn=1
for (j = first; j < pastlast; j++) { for (j = first; j < pastlast; j++) {
builddir=KVM "/b" jn builddir=KVM "/b" jn
...@@ -371,25 +378,28 @@ END { ...@@ -371,25 +378,28 @@ END {
njobs = i; njobs = i;
nc = ncpus; nc = ncpus;
first = 0; first = 0;
batchnum = 1;
# Each pass through the following loop considers one test. # Each pass through the following loop considers one test.
for (i = 0; i < njobs; i++) { for (i = 0; i < njobs; i++) {
if (ncpus == 0) { if (ncpus == 0) {
# Sequential test specified, each test its own batch. # Sequential test specified, each test its own batch.
dump(i, i + 1); dump(i, i + 1, batchnum);
first = i; first = i;
batchnum++;
} else if (nc < cpus[i] && i != 0) { } else if (nc < cpus[i] && i != 0) {
# Out of CPUs, dump out a batch. # Out of CPUs, dump out a batch.
dump(first, i); dump(first, i, batchnum);
first = i; first = i;
nc = ncpus; nc = ncpus;
batchnum++;
} }
# Account for the CPUs needed by the current test. # Account for the CPUs needed by the current test.
nc -= cpus[i]; nc -= cpus[i];
} }
# Dump the last batch. # Dump the last batch.
if (ncpus != 0) if (ncpus != 0)
dump(first, i); dump(first, i, batchnum);
}' >> $T/script }' >> $T/script
cat << ___EOF___ >> $T/script cat << ___EOF___ >> $T/script
......
tools/testing/selftests/rcutorture/bin/parse-console.sh
View file @ 3104fb3d
...@@ -24,9 +24,6 @@ ...@@ -24,9 +24,6 @@
# #
# Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com> # Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
T=/tmp/abat-chk-badness.sh.$$
trap 'rm -f $T' 0
file="$1" file="$1"
title="$2" title="$2"
...@@ -36,9 +33,41 @@ if grep -Pq '\x00' < $file ...@@ -36,9 +33,41 @@ if grep -Pq '\x00' < $file
then then
print_warning Console output contains nul bytes, old qemu still running? print_warning Console output contains nul bytes, old qemu still running?
fi fi
egrep 'Badness|WARNING:|Warn|BUG|===========|Call Trace:|Oops:|Stall ended before state dump start' < $file | grep -v 'ODEBUG: ' | grep -v 'Warning: unable to open an initial console' > $T egrep 'Badness|WARNING:|Warn|BUG|===========|Call Trace:|Oops:|detected stalls on CPUs/tasks:|Stall ended before state dump start' < $file | grep -v 'ODEBUG: ' | grep -v 'Warning: unable to open an initial console' > $1.diags
if test -s $T if test -s $1.diags
then then
print_warning Assertion failure in $file $title print_warning Assertion failure in $file $title
cat $T # cat $1.diags
summary=""
n_badness=`grep -c Badness $1`
if test "$n_badness" -ne 0
then
summary="$summary Badness: $n_badness"
fi
n_warn=`grep -v 'Warning: unable to open an initial console' $1 | egrep -c 'WARNING:|Warn'`
if test "$n_warn" -ne 0
then
summary="$summary Warnings: $n_warn"
fi
n_bugs=`egrep -c 'BUG|Oops:' $1`
if test "$n_bugs" -ne 0
then
summary="$summary Bugs: $n_bugs"
fi
n_calltrace=`grep -c 'Call Trace:' $1`
if test "$n_calltrace" -ne 0
then
summary="$summary Call Traces: $n_calltrace"
fi
n_lockdep=`grep -c =========== $1`
if test "$n_badness" -ne 0
then
summary="$summary lockdep: $n_badness"
fi
n_stalls=`egrep -c 'detected stalls on CPUs/tasks:|Stall ended before state dump start' $1`
if test "$n_stalls" -ne 0
then
summary="$summary Stalls: $n_stalls"
fi
print_warning Summary: $summary
fi fi
tools/testing/selftests/rcutorture/doc/TINY_RCU.txt
View file @ 3104fb3d
...@@ -20,7 +20,6 @@ CONFIG_PROVE_RCU ...@@ -20,7 +20,6 @@ CONFIG_PROVE_RCU
CONFIG_NO_HZ_FULL_SYSIDLE CONFIG_NO_HZ_FULL_SYSIDLE
CONFIG_RCU_NOCB_CPU CONFIG_RCU_NOCB_CPU
CONFIG_RCU_USER_QS
Meaningless for TINY_RCU. Meaningless for TINY_RCU.
......
tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt
View file @ 3104fb3d
...@@ -72,10 +72,6 @@ CONFIG_RCU_TORTURE_TEST_RUNNABLE ...@@ -72,10 +72,6 @@ CONFIG_RCU_TORTURE_TEST_RUNNABLE
Always used in KVM testing. Always used in KVM testing.
CONFIG_RCU_USER_QS
Redundant with CONFIG_NO_HZ_FULL.
CONFIG_PREEMPT_RCU CONFIG_PREEMPT_RCU
CONFIG_TREE_RCU CONFIG_TREE_RCU
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7