|
|
|
@@ -114,6 +114,23 @@ static struct tk_fast tk_fast_raw ____cacheline_aligned = {
|
|
|
|
|
.base[1] = FAST_TK_INIT,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Multigrain timestamps require tracking the latest fine-grained timestamp
|
|
|
|
|
* that has been issued, and never returning a coarse-grained timestamp that is
|
|
|
|
|
* earlier than that value.
|
|
|
|
|
*
|
|
|
|
|
* mg_floor represents the latest fine-grained time that has been handed out as
|
|
|
|
|
* a file timestamp on the system. This is tracked as a monotonic ktime_t, and
|
|
|
|
|
* converted to a realtime clock value on an as-needed basis.
|
|
|
|
|
*
|
|
|
|
|
* Maintaining mg_floor ensures the multigrain interfaces never issue a
|
|
|
|
|
* timestamp earlier than one that has been previously issued.
|
|
|
|
|
*
|
|
|
|
|
* The exception to this rule is when there is a backward realtime clock jump. If
|
|
|
|
|
* such an event occurs, a timestamp can appear to be earlier than a previous one.
|
|
|
|
|
*/
|
|
|
|
|
static __cacheline_aligned_in_smp atomic64_t mg_floor;
|
|
|
|
|
|
|
|
|
|
static inline void tk_normalize_xtime(struct timekeeper *tk)
|
|
|
|
|
{
|
|
|
|
|
while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
|
|
|
|
@@ -2408,6 +2425,94 @@ void ktime_get_coarse_real_ts64(struct timespec64 *ts)
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(ktime_get_coarse_real_ts64);
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* ktime_get_coarse_real_ts64_mg - return latter of coarse grained time or floor
|
|
|
|
|
* @ts: timespec64 to be filled
|
|
|
|
|
*
|
|
|
|
|
* Fetch the global mg_floor value, convert it to realtime and compare it
|
|
|
|
|
* to the current coarse-grained time. Fill @ts with whichever is
|
|
|
|
|
* latest. Note that this is a filesystem-specific interface and should be
|
|
|
|
|
* avoided outside of that context.
|
|
|
|
|
*/
|
|
|
|
|
void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts)
|
|
|
|
|
{
|
|
|
|
|
struct timekeeper *tk = &tk_core.timekeeper;
|
|
|
|
|
u64 floor = atomic64_read(&mg_floor);
|
|
|
|
|
ktime_t f_real, offset, coarse;
|
|
|
|
|
unsigned int seq;
|
|
|
|
|
|
|
|
|
|
do {
|
|
|
|
|
seq = read_seqcount_begin(&tk_core.seq);
|
|
|
|
|
*ts = tk_xtime(tk);
|
|
|
|
|
offset = tk_core.timekeeper.offs_real;
|
|
|
|
|
} while (read_seqcount_retry(&tk_core.seq, seq));
|
|
|
|
|
|
|
|
|
|
coarse = timespec64_to_ktime(*ts);
|
|
|
|
|
f_real = ktime_add(floor, offset);
|
|
|
|
|
if (ktime_after(f_real, coarse))
|
|
|
|
|
*ts = ktime_to_timespec64(f_real);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* ktime_get_real_ts64_mg - attempt to update floor value and return result
|
|
|
|
|
* @ts: pointer to the timespec to be set
|
|
|
|
|
*
|
|
|
|
|
* Get a monotonic fine-grained time value and attempt to swap it into
|
|
|
|
|
* mg_floor. If that succeeds then accept the new floor value. If it fails
|
|
|
|
|
* then another task raced in during the interim time and updated the
|
|
|
|
|
* floor. Since any update to the floor must be later than the previous
|
|
|
|
|
* floor, either outcome is acceptable.
|
|
|
|
|
*
|
|
|
|
|
* Typically this will be called after calling ktime_get_coarse_real_ts64_mg(),
|
|
|
|
|
* and determining that the resulting coarse-grained timestamp did not effect
|
|
|
|
|
* a change in ctime. Any more recent floor value would effect a change to
|
|
|
|
|
* ctime, so there is no need to retry the atomic64_try_cmpxchg() on failure.
|
|
|
|
|
*
|
|
|
|
|
* @ts will be filled with the latest floor value, regardless of the outcome of
|
|
|
|
|
* the cmpxchg. Note that this is a filesystem specific interface and should be
|
|
|
|
|
* avoided outside of that context.
|
|
|
|
|
*/
|
|
|
|
|
void ktime_get_real_ts64_mg(struct timespec64 *ts)
|
|
|
|
|
{
|
|
|
|
|
struct timekeeper *tk = &tk_core.timekeeper;
|
|
|
|
|
ktime_t old = atomic64_read(&mg_floor);
|
|
|
|
|
ktime_t offset, mono;
|
|
|
|
|
unsigned int seq;
|
|
|
|
|
u64 nsecs;
|
|
|
|
|
|
|
|
|
|
do {
|
|
|
|
|
seq = read_seqcount_begin(&tk_core.seq);
|
|
|
|
|
|
|
|
|
|
ts->tv_sec = tk->xtime_sec;
|
|
|
|
|
mono = tk->tkr_mono.base;
|
|
|
|
|
nsecs = timekeeping_get_ns(&tk->tkr_mono);
|
|
|
|
|
offset = tk_core.timekeeper.offs_real;
|
|
|
|
|
} while (read_seqcount_retry(&tk_core.seq, seq));
|
|
|
|
|
|
|
|
|
|
mono = ktime_add_ns(mono, nsecs);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Attempt to update the floor with the new time value. As any
|
|
|
|
|
* update must be later then the existing floor, and would effect
|
|
|
|
|
* a change to ctime from the perspective of the current task,
|
|
|
|
|
* accept the resulting floor value regardless of the outcome of
|
|
|
|
|
* the swap.
|
|
|
|
|
*/
|
|
|
|
|
if (atomic64_try_cmpxchg(&mg_floor, &old, mono)) {
|
|
|
|
|
ts->tv_nsec = 0;
|
|
|
|
|
timespec64_add_ns(ts, nsecs);
|
|
|
|
|
timekeeping_inc_mg_floor_swaps();
|
|
|
|
|
} else {
|
|
|
|
|
/*
|
|
|
|
|
* Another task changed mg_floor since "old" was fetched.
|
|
|
|
|
* "old" has been updated with the latest value of "mg_floor".
|
|
|
|
|
* That value is newer than the previous floor value, which
|
|
|
|
|
* is enough to effect a change to ctime. Accept it.
|
|
|
|
|
*/
|
|
|
|
|
*ts = ktime_to_timespec64(ktime_add(old, offset));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void ktime_get_coarse_ts64(struct timespec64 *ts)
|
|
|
|
|
{
|
|
|
|
|
struct timekeeper *tk = &tk_core.timekeeper;
|
|
|
|
|
Thomas Gleixner