Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : #ifndef _LINUX_MATH64_H
3 : #define _LINUX_MATH64_H
4 :
5 : #include <linux/types.h>
6 : #include <linux/math.h>
7 : #include <vdso/math64.h>
8 : #include <asm/div64.h>
9 :
10 : #if BITS_PER_LONG == 64
11 :
12 : #define div64_long(x, y) div64_s64((x), (y))
13 : #define div64_ul(x, y) div64_u64((x), (y))
14 :
15 : /**
16 : * div_u64_rem - unsigned 64bit divide with 32bit divisor with remainder
17 : * @dividend: unsigned 64bit dividend
18 : * @divisor: unsigned 32bit divisor
19 : * @remainder: pointer to unsigned 32bit remainder
20 : *
21 : * Return: sets ``*remainder``, then returns dividend / divisor
22 : *
23 : * This is commonly provided by 32bit archs to provide an optimized 64bit
24 : * divide.
25 : */
26 : static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
27 : {
28 792 : *remainder = dividend % divisor;
29 789 : return dividend / divisor;
30 : }
31 :
32 : /**
33 : * div_s64_rem - signed 64bit divide with 32bit divisor with remainder
34 : * @dividend: signed 64bit dividend
35 : * @divisor: signed 32bit divisor
36 : * @remainder: pointer to signed 32bit remainder
37 : *
38 : * Return: sets ``*remainder``, then returns dividend / divisor
39 : */
40 : static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder)
41 : {
42 0 : *remainder = dividend % divisor;
43 0 : return dividend / divisor;
44 : }
45 :
46 : /**
47 : * div64_u64_rem - unsigned 64bit divide with 64bit divisor and remainder
48 : * @dividend: unsigned 64bit dividend
49 : * @divisor: unsigned 64bit divisor
50 : * @remainder: pointer to unsigned 64bit remainder
51 : *
52 : * Return: sets ``*remainder``, then returns dividend / divisor
53 : */
54 : static inline u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder)
55 : {
56 4407615 : *remainder = dividend % divisor;
57 0 : return dividend / divisor;
58 : }
59 :
60 : /**
61 : * div64_u64 - unsigned 64bit divide with 64bit divisor
62 : * @dividend: unsigned 64bit dividend
63 : * @divisor: unsigned 64bit divisor
64 : *
65 : * Return: dividend / divisor
66 : */
67 : static inline u64 div64_u64(u64 dividend, u64 divisor)
68 : {
69 49205 : return dividend / divisor;
70 : }
71 :
72 : /**
73 : * div64_s64 - signed 64bit divide with 64bit divisor
74 : * @dividend: signed 64bit dividend
75 : * @divisor: signed 64bit divisor
76 : *
77 : * Return: dividend / divisor
78 : */
79 : static inline s64 div64_s64(s64 dividend, s64 divisor)
80 : {
81 0 : return dividend / divisor;
82 : }
83 :
84 : #elif BITS_PER_LONG == 32
85 :
86 : #define div64_long(x, y) div_s64((x), (y))
87 : #define div64_ul(x, y) div_u64((x), (y))
88 :
89 : #ifndef div_u64_rem
90 : static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
91 : {
92 : *remainder = do_div(dividend, divisor);
93 : return dividend;
94 : }
95 : #endif
96 :
97 : #ifndef div_s64_rem
98 : extern s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder);
99 : #endif
100 :
101 : #ifndef div64_u64_rem
102 : extern u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder);
103 : #endif
104 :
105 : #ifndef div64_u64
106 : extern u64 div64_u64(u64 dividend, u64 divisor);
107 : #endif
108 :
109 : #ifndef div64_s64
110 : extern s64 div64_s64(s64 dividend, s64 divisor);
111 : #endif
112 :
113 : #endif /* BITS_PER_LONG */
114 :
115 : /**
116 : * div_u64 - unsigned 64bit divide with 32bit divisor
117 : * @dividend: unsigned 64bit dividend
118 : * @divisor: unsigned 32bit divisor
119 : *
120 : * This is the most common 64bit divide and should be used if possible,
121 : * as many 32bit archs can optimize this variant better than a full 64bit
122 : * divide.
123 : *
124 : * Return: dividend / divisor
125 : */
126 : #ifndef div_u64
127 : static inline u64 div_u64(u64 dividend, u32 divisor)
128 : {
129 : u32 remainder;
130 791 : return div_u64_rem(dividend, divisor, &remainder);
131 : }
132 : #endif
133 :
134 : /**
135 : * div_s64 - signed 64bit divide with 32bit divisor
136 : * @dividend: signed 64bit dividend
137 : * @divisor: signed 32bit divisor
138 : *
139 : * Return: dividend / divisor
140 : */
141 : #ifndef div_s64
142 : static inline s64 div_s64(s64 dividend, s32 divisor)
143 : {
144 : s32 remainder;
145 0 : return div_s64_rem(dividend, divisor, &remainder);
146 : }
147 : #endif
148 :
149 : u32 iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder);
150 :
151 : #ifndef mul_u32_u32
152 : /*
153 : * Many a GCC version messes this up and generates a 64x64 mult :-(
154 : */
155 : static inline u64 mul_u32_u32(u32 a, u32 b)
156 : {
157 19620 : return (u64)a * b;
158 : }
159 : #endif
160 :
161 : #if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__)
162 :
163 : #ifndef mul_u64_u32_shr
164 : static __always_inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift)
165 : {
166 : return (u64)(((unsigned __int128)a * mul) >> shift);
167 : }
168 : #endif /* mul_u64_u32_shr */
169 :
170 : #ifndef mul_u64_u64_shr
171 : static inline u64 mul_u64_u64_shr(u64 a, u64 mul, unsigned int shift)
172 : {
173 : return (u64)(((unsigned __int128)a * mul) >> shift);
174 : }
175 : #endif /* mul_u64_u64_shr */
176 :
177 : #else
178 :
179 : #ifndef mul_u64_u32_shr
180 : static __always_inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift)
181 : {
182 : u32 ah, al;
183 : u64 ret;
184 :
185 617 : al = a;
186 617 : ah = a >> 32;
187 :
188 617 : ret = mul_u32_u32(al, mul) >> shift;
189 617 : if (ah)
190 0 : ret += mul_u32_u32(ah, mul) << (32 - shift);
191 :
192 : return ret;
193 : }
194 : #endif /* mul_u64_u32_shr */
195 :
196 : #ifndef mul_u64_u64_shr
197 : static inline u64 mul_u64_u64_shr(u64 a, u64 b, unsigned int shift)
198 : {
199 : union {
200 : u64 ll;
201 : struct {
202 : #ifdef __BIG_ENDIAN
203 : u32 high, low;
204 : #else
205 : u32 low, high;
206 : #endif
207 : } l;
208 : } rl, rm, rn, rh, a0, b0;
209 : u64 c;
210 :
211 : a0.ll = a;
212 : b0.ll = b;
213 :
214 : rl.ll = mul_u32_u32(a0.l.low, b0.l.low);
215 : rm.ll = mul_u32_u32(a0.l.low, b0.l.high);
216 : rn.ll = mul_u32_u32(a0.l.high, b0.l.low);
217 : rh.ll = mul_u32_u32(a0.l.high, b0.l.high);
218 :
219 : /*
220 : * Each of these lines computes a 64-bit intermediate result into "c",
221 : * starting at bits 32-95. The low 32-bits go into the result of the
222 : * multiplication, the high 32-bits are carried into the next step.
223 : */
224 : rl.l.high = c = (u64)rl.l.high + rm.l.low + rn.l.low;
225 : rh.l.low = c = (c >> 32) + rm.l.high + rn.l.high + rh.l.low;
226 : rh.l.high = (c >> 32) + rh.l.high;
227 :
228 : /*
229 : * The 128-bit result of the multiplication is in rl.ll and rh.ll,
230 : * shift it right and throw away the high part of the result.
231 : */
232 : if (shift == 0)
233 : return rl.ll;
234 : if (shift < 64)
235 : return (rl.ll >> shift) | (rh.ll << (64 - shift));
236 : return rh.ll >> (shift & 63);
237 : }
238 : #endif /* mul_u64_u64_shr */
239 :
240 : #endif
241 :
242 : #ifndef mul_s64_u64_shr
243 : static inline u64 mul_s64_u64_shr(s64 a, u64 b, unsigned int shift)
244 : {
245 : u64 ret;
246 :
247 : /*
248 : * Extract the sign before the multiplication and put it back
249 : * afterwards if needed.
250 : */
251 : ret = mul_u64_u64_shr(abs(a), b, shift);
252 :
253 : if (a < 0)
254 : ret = -((s64) ret);
255 :
256 : return ret;
257 : }
258 : #endif /* mul_s64_u64_shr */
259 :
260 : #ifndef mul_u64_u32_div
261 : static inline u64 mul_u64_u32_div(u64 a, u32 mul, u32 divisor)
262 : {
263 : union {
264 : u64 ll;
265 : struct {
266 : #ifdef __BIG_ENDIAN
267 : u32 high, low;
268 : #else
269 : u32 low, high;
270 : #endif
271 : } l;
272 : } u, rl, rh;
273 :
274 : u.ll = a;
275 : rl.ll = mul_u32_u32(u.l.low, mul);
276 : rh.ll = mul_u32_u32(u.l.high, mul) + rl.l.high;
277 :
278 : /* Bits 32-63 of the result will be in rh.l.low. */
279 : rl.l.high = do_div(rh.ll, divisor);
280 :
281 : /* Bits 0-31 of the result will be in rl.l.low. */
282 : do_div(rl.ll, divisor);
283 :
284 : rl.l.high = rh.l.low;
285 : return rl.ll;
286 : }
287 : #endif /* mul_u64_u32_div */
288 :
289 : u64 mul_u64_u64_div_u64(u64 a, u64 mul, u64 div);
290 :
291 : /**
292 : * DIV64_U64_ROUND_UP - unsigned 64bit divide with 64bit divisor rounded up
293 : * @ll: unsigned 64bit dividend
294 : * @d: unsigned 64bit divisor
295 : *
296 : * Divide unsigned 64bit dividend by unsigned 64bit divisor
297 : * and round up.
298 : *
299 : * Return: dividend / divisor rounded up
300 : */
301 : #define DIV64_U64_ROUND_UP(ll, d) \
302 : ({ u64 _tmp = (d); div64_u64((ll) + _tmp - 1, _tmp); })
303 :
304 : /**
305 : * DIV64_U64_ROUND_CLOSEST - unsigned 64bit divide with 64bit divisor rounded to nearest integer
306 : * @dividend: unsigned 64bit dividend
307 : * @divisor: unsigned 64bit divisor
308 : *
309 : * Divide unsigned 64bit dividend by unsigned 64bit divisor
310 : * and round to closest integer.
311 : *
312 : * Return: dividend / divisor rounded to nearest integer
313 : */
314 : #define DIV64_U64_ROUND_CLOSEST(dividend, divisor) \
315 : ({ u64 _tmp = (divisor); div64_u64((dividend) + _tmp / 2, _tmp); })
316 :
317 : /**
318 : * DIV_U64_ROUND_CLOSEST - unsigned 64bit divide with 32bit divisor rounded to nearest integer
319 : * @dividend: unsigned 64bit dividend
320 : * @divisor: unsigned 32bit divisor
321 : *
322 : * Divide unsigned 64bit dividend by unsigned 32bit divisor
323 : * and round to closest integer.
324 : *
325 : * Return: dividend / divisor rounded to nearest integer
326 : */
327 : #define DIV_U64_ROUND_CLOSEST(dividend, divisor) \
328 : ({ u32 _tmp = (divisor); div_u64((u64)(dividend) + _tmp / 2, _tmp); })
329 :
330 : /**
331 : * DIV_S64_ROUND_CLOSEST - signed 64bit divide with 32bit divisor rounded to nearest integer
332 : * @dividend: signed 64bit dividend
333 : * @divisor: signed 32bit divisor
334 : *
335 : * Divide signed 64bit dividend by signed 32bit divisor
336 : * and round to closest integer.
337 : *
338 : * Return: dividend / divisor rounded to nearest integer
339 : */
340 : #define DIV_S64_ROUND_CLOSEST(dividend, divisor)( \
341 : { \
342 : s64 __x = (dividend); \
343 : s32 __d = (divisor); \
344 : ((__x > 0) == (__d > 0)) ? \
345 : div_s64((__x + (__d / 2)), __d) : \
346 : div_s64((__x - (__d / 2)), __d); \
347 : } \
348 : )
349 : #endif /* _LINUX_MATH64_H */
|
