summaryrefslogtreecommitdiffstats
path: root/js/src/wasm/WasmBCFrame.cpp
blob: 4e04875233b19d57326fd54a502316c336a4626b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
 * vim: set ts=8 sts=2 et sw=2 tw=80:
 *
 * Copyright 2016 Mozilla Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "wasm/WasmBCFrame.h"

#include "wasm/WasmBaselineCompile.h"  // For BaseLocalIter
#include "wasm/WasmBCClass.h"

#include "jit/MacroAssembler-inl.h"
#include "wasm/WasmBCClass-inl.h"
#include "wasm/WasmBCCodegen-inl.h"
#include "wasm/WasmBCRegDefs-inl.h"
#include "wasm/WasmBCRegMgmt-inl.h"
#include "wasm/WasmBCStkMgmt-inl.h"

namespace js {
namespace wasm {

//////////////////////////////////////////////////////////////////////////////
//
// BaseLocalIter methods.

BaseLocalIter::BaseLocalIter(const ValTypeVector& locals,
                             const ArgTypeVector& args, bool debugEnabled)
    : locals_(locals),
      args_(args),
      argsIter_(args_),
      index_(0),
      frameSize_(0),
      nextFrameSize_(debugEnabled ? DebugFrame::offsetOfFrame() : 0),
      frameOffset_(INT32_MAX),
      stackResultPointerOffset_(INT32_MAX),
      mirType_(MIRType::Undefined),
      done_(false) {
  MOZ_ASSERT(args.lengthWithoutStackResults() <= locals.length());
  settle();
}

int32_t BaseLocalIter::pushLocal(size_t nbytes) {
  MOZ_ASSERT(nbytes % 4 == 0 && nbytes <= 16);
  nextFrameSize_ = AlignBytes(frameSize_, nbytes) + nbytes;
  return nextFrameSize_;  // Locals grow down so capture base address.
}

void BaseLocalIter::settle() {
  MOZ_ASSERT(!done_);
  frameSize_ = nextFrameSize_;

  if (!argsIter_.done()) {
    mirType_ = argsIter_.mirType();
    MIRType concreteType = mirType_;
    switch (mirType_) {
      case MIRType::StackResults:
        // The pointer to stack results is handled like any other argument:
        // either addressed in place if it is passed on the stack, or we spill
        // it in the frame if it's in a register.
        MOZ_ASSERT(args_.isSyntheticStackResultPointerArg(index_));
        concreteType = MIRType::Pointer;
        [[fallthrough]];
      case MIRType::Int32:
      case MIRType::Int64:
      case MIRType::Double:
      case MIRType::Float32:
      case MIRType::WasmAnyRef:
#ifdef ENABLE_WASM_SIMD
      case MIRType::Simd128:
#endif
        if (argsIter_->argInRegister()) {
          frameOffset_ = pushLocal(MIRTypeToSize(concreteType));
        } else {
          frameOffset_ = -(argsIter_->offsetFromArgBase() + sizeof(Frame));
        }
        break;
      default:
        MOZ_CRASH("Argument type");
    }
    if (mirType_ == MIRType::StackResults) {
      stackResultPointerOffset_ = frameOffset();
      // Advance past the synthetic stack result pointer argument and fall
      // through to the next case.
      argsIter_++;
      frameSize_ = nextFrameSize_;
      MOZ_ASSERT(argsIter_.done());
    } else {
      return;
    }
  }

  if (index_ < locals_.length()) {
    switch (locals_[index_].kind()) {
      case ValType::I32:
      case ValType::I64:
      case ValType::F32:
      case ValType::F64:
#ifdef ENABLE_WASM_SIMD
      case ValType::V128:
#endif
      case ValType::Ref:
        mirType_ = locals_[index_].toMIRType();
        frameOffset_ = pushLocal(MIRTypeToSize(mirType_));
        break;
      default:
        MOZ_CRASH("Compiler bug: Unexpected local type");
    }
    return;
  }

  done_ = true;
}

void BaseLocalIter::operator++(int) {
  MOZ_ASSERT(!done_);
  index_++;
  if (!argsIter_.done()) {
    argsIter_++;
  }
  settle();
}

//////////////////////////////////////////////////////////////////////////////
//
// Stack map methods.

bool BaseCompiler::createStackMap(const char* who) {
  const ExitStubMapVector noExtras;
  return stackMapGenerator_.createStackMap(who, noExtras, masm.currentOffset(),
                                           HasDebugFrameWithLiveRefs::No, stk_);
}

bool BaseCompiler::createStackMap(const char* who, CodeOffset assemblerOffset) {
  const ExitStubMapVector noExtras;
  return stackMapGenerator_.createStackMap(who, noExtras,
                                           assemblerOffset.offset(),
                                           HasDebugFrameWithLiveRefs::No, stk_);
}

bool BaseCompiler::createStackMap(
    const char* who, HasDebugFrameWithLiveRefs debugFrameWithLiveRefs) {
  const ExitStubMapVector noExtras;
  return stackMapGenerator_.createStackMap(who, noExtras, masm.currentOffset(),
                                           debugFrameWithLiveRefs, stk_);
}

bool BaseCompiler::createStackMap(
    const char* who, const ExitStubMapVector& extras, uint32_t assemblerOffset,
    HasDebugFrameWithLiveRefs debugFrameWithLiveRefs) {
  return stackMapGenerator_.createStackMap(who, extras, assemblerOffset,
                                           debugFrameWithLiveRefs, stk_);
}

bool MachineStackTracker::cloneTo(MachineStackTracker* dst) {
  MOZ_ASSERT(dst->vec_.empty());
  if (!dst->vec_.appendAll(vec_)) {
    return false;
  }
  dst->numPtrs_ = numPtrs_;
  return true;
}

bool StackMapGenerator::generateStackmapEntriesForTrapExit(
    const ArgTypeVector& args, ExitStubMapVector* extras) {
  return GenerateStackmapEntriesForTrapExit(args, trapExitLayout_,
                                            trapExitLayoutNumWords_, extras);
}

bool StackMapGenerator::createStackMap(
    const char* who, const ExitStubMapVector& extras, uint32_t assemblerOffset,
    HasDebugFrameWithLiveRefs debugFrameWithLiveRefs, const StkVector& stk) {
  size_t countedPointers = machineStackTracker.numPtrs() + memRefsOnStk;
#ifndef DEBUG
  // An important optimization.  If there are obviously no pointers, as
  // we expect in the majority of cases, exit quickly.
  if (countedPointers == 0 &&
      debugFrameWithLiveRefs == HasDebugFrameWithLiveRefs::No) {
    // We can skip creating the map if there are no |true| elements in
    // |extras|.
    bool extrasHasRef = false;
    for (bool b : extras) {
      if (b) {
        extrasHasRef = true;
        break;
      }
    }
    if (!extrasHasRef) {
      return true;
    }
  }
#else
  // In the debug case, create the stackmap regardless, and cross-check
  // the pointer-counting below.  We expect the final map to have
  // |countedPointers| in total.  This doesn't include those in the
  // DebugFrame, but they do not appear in the map's bitmap.  Note that
  // |countedPointers| is debug-only from this point onwards.
  for (bool b : extras) {
    countedPointers += (b ? 1 : 0);
  }
#endif

  // Start with the frame-setup map, and add operand-stack information to
  // that.  augmentedMst holds live data only within individual calls to
  // createStackMap.
  augmentedMst.clear();
  if (!machineStackTracker.cloneTo(&augmentedMst)) {
    return false;
  }

  // At this point, augmentedMst only contains entries covering the
  // incoming argument area (if any) and for the area allocated by this
  // function's prologue.  We now need to calculate how far the machine's
  // stack pointer is below where it was at the start of the body.  But we
  // must take care not to include any words pushed as arguments to an
  // upcoming function call, since those words belong to the stackmap of
  // the callee, not to the stackmap of this function.  Any alignment padding
  // for the args also belongs to the callee.
  //
  // The only padding belonging to the stackmap of this function is that
  // required to align the upcoming frame. This is accounted for where
  // framePushedExcludingOutboundCallArgs is set, in startCallArgs(), and is
  // comprised of just one component:
  //
  // * call->frameAlignAdjustment
  Maybe<uint32_t> framePushedExcludingArgs;
  if (framePushedAtEntryToBody.isNothing()) {
    // Still in the prologue.  framePushedExcludingArgs remains Nothing.
    MOZ_ASSERT(framePushedExcludingOutboundCallArgs.isNothing());
  } else {
    // In the body.
    MOZ_ASSERT(masm_.framePushed() >= framePushedAtEntryToBody.value());
    if (framePushedExcludingOutboundCallArgs.isSome()) {
      // In the body, and we've potentially pushed some args onto the stack.
      // We must ignore them when sizing the stackmap.
      MOZ_ASSERT(masm_.framePushed() >=
                 framePushedExcludingOutboundCallArgs.value());
      MOZ_ASSERT(framePushedExcludingOutboundCallArgs.value() >=
                 framePushedAtEntryToBody.value());
      framePushedExcludingArgs =
          Some(framePushedExcludingOutboundCallArgs.value());
    } else {
      // In the body, but not with call args on the stack.  The stackmap
      // must be sized so as to extend all the way "down" to
      // masm_.framePushed().
      framePushedExcludingArgs = Some(masm_.framePushed());
    }
  }

  if (framePushedExcludingArgs.isSome()) {
    uint32_t bodyPushedBytes =
        framePushedExcludingArgs.value() - framePushedAtEntryToBody.value();
    MOZ_ASSERT(0 == bodyPushedBytes % sizeof(void*));
    if (!augmentedMst.pushNonGCPointers(bodyPushedBytes / sizeof(void*))) {
      return false;
    }
  }

  // Scan the operand stack, marking pointers in the just-added new
  // section.
  MOZ_ASSERT_IF(framePushedAtEntryToBody.isNothing(), stk.empty());
  MOZ_ASSERT_IF(framePushedExcludingArgs.isNothing(), stk.empty());

  for (const Stk& v : stk) {
#ifndef DEBUG
    // We don't track roots in registers, per rationale below, so if this
    // doesn't hold, something is seriously wrong, and we're likely to get a
    // GC-related crash.
    MOZ_RELEASE_ASSERT(v.kind() != Stk::RegisterRef);
    if (v.kind() != Stk::MemRef) {
      continue;
    }
#else
    // Take the opportunity to check everything we reasonably can about
    // operand stack elements.
    switch (v.kind()) {
      case Stk::MemI32:
      case Stk::MemI64:
      case Stk::MemF32:
      case Stk::MemF64:
      case Stk::ConstI32:
      case Stk::ConstI64:
      case Stk::ConstF32:
      case Stk::ConstF64:
#  ifdef ENABLE_WASM_SIMD
      case Stk::MemV128:
      case Stk::ConstV128:
#  endif
        // All of these have uninteresting type.
        continue;
      case Stk::LocalI32:
      case Stk::LocalI64:
      case Stk::LocalF32:
      case Stk::LocalF64:
#  ifdef ENABLE_WASM_SIMD
      case Stk::LocalV128:
#  endif
        // These also have uninteresting type.  Check that they live in the
        // section of stack set up by beginFunction().  The unguarded use of
        // |value()| here is safe due to the assertion above this loop.
        MOZ_ASSERT(v.offs() <= framePushedAtEntryToBody.value());
        continue;
      case Stk::RegisterI32:
      case Stk::RegisterI64:
      case Stk::RegisterF32:
      case Stk::RegisterF64:
#  ifdef ENABLE_WASM_SIMD
      case Stk::RegisterV128:
#  endif
        // These also have uninteresting type, but more to the point: all
        // registers holding live values should have been flushed to the
        // machine stack immediately prior to the instruction to which this
        // stackmap pertains.  So these can't happen.
        MOZ_CRASH("createStackMap: operand stack has Register-non-Ref");
      case Stk::MemRef:
        // This is the only case we care about.  We'll handle it after the
        // switch.
        break;
      case Stk::LocalRef:
        // We need the stackmap to mention this pointer, but it should
        // already be in the machineStackTracker section created by
        // beginFunction().
        MOZ_ASSERT(v.offs() <= framePushedAtEntryToBody.value());
        continue;
      case Stk::ConstRef:
        // This can currently only be a null pointer.
        MOZ_ASSERT(v.refval() == 0);
        continue;
      case Stk::RegisterRef:
        // This can't happen, per rationale above.
        MOZ_CRASH("createStackMap: operand stack contains RegisterRef");
      default:
        MOZ_CRASH("createStackMap: unknown operand stack element");
    }
#endif
    // v.offs() holds masm.framePushed() at the point immediately after it
    // was pushed on the stack.  Since it's still on the stack,
    // masm.framePushed() can't be less.
    MOZ_ASSERT(v.offs() <= framePushedExcludingArgs.value());
    uint32_t offsFromMapLowest = framePushedExcludingArgs.value() - v.offs();
    MOZ_ASSERT(0 == offsFromMapLowest % sizeof(void*));
    augmentedMst.setGCPointer(offsFromMapLowest / sizeof(void*));
  }

  MOZ_ASSERT(numStackArgBytes % sizeof(void*) == 0);
  const size_t numStackArgWords = numStackArgBytes / sizeof(void*);
  const size_t numStackArgPaddingBytes =
      AlignStackArgAreaSize(numStackArgBytes) - numStackArgBytes;
  const size_t numStackArgPaddingWords =
      numStackArgPaddingBytes / sizeof(void*);

  // Create the final StackMap.  The initial map is zeroed out, so there's
  // no need to write zero bits in it.
  const uint32_t extraWords = extras.length();
  const uint32_t augmentedMstWords = augmentedMst.length();
  const uint32_t numMappedWords =
      numStackArgPaddingWords + extraWords + augmentedMstWords;
  StackMap* stackMap = StackMap::create(numMappedWords);
  if (!stackMap) {
    return false;
  }

  {
    // First the exit stub extra words, if any.
    uint32_t i = 0;
    for (bool b : extras) {
      if (b) {
        stackMap->set(i, StackMap::Kind::AnyRef);
      }
      i++;
    }
  }
  {
    // Followed by the "main" part of the map.
    //
    // This is really just a bit-array copy, so it is reasonable to ask
    // whether the representation of MachineStackTracker could be made more
    // similar to that of StackMap, so that the copy could be done with
    // `memcpy`.  Unfortunately it's not so simple; see comment on `class
    // MachineStackTracker` for details.
    MachineStackTracker::Iter iter(augmentedMst);
    while (true) {
      size_t i = iter.get();
      if (i == MachineStackTracker::Iter::FINISHED) {
        break;
      }
      stackMap->set(extraWords + i, StackMap::Kind::AnyRef);
    }
  }

  stackMap->setExitStubWords(extraWords);

  // Record in the map, how far down from the highest address the Frame* is.
  // Take the opportunity to check that we haven't marked any part of the
  // Frame itself as a pointer.
  stackMap->setFrameOffsetFromTop(numStackArgPaddingWords + numStackArgWords +
                                  sizeof(Frame) / sizeof(void*));
#ifdef DEBUG
  for (uint32_t i = 0; i < sizeof(Frame) / sizeof(void*); i++) {
    MOZ_ASSERT(stackMap->get(stackMap->header.numMappedWords -
                             stackMap->header.frameOffsetFromTop + i) ==
               StackMap::Kind::POD);
  }
#endif

  // Note the presence of a DebugFrame with live pointers, if any.
  if (debugFrameWithLiveRefs != HasDebugFrameWithLiveRefs::No) {
    stackMap->setHasDebugFrameWithLiveRefs();
  }

  // Add the completed map to the running collection thereof.
  if (!stackMaps_->add((uint8_t*)(uintptr_t)assemblerOffset, stackMap)) {
    stackMap->destroy();
    return false;
  }

#ifdef DEBUG
  {
    // Crosscheck the map pointer counting.
    uint32_t nw = stackMap->header.numMappedWords;
    uint32_t np = 0;
    for (uint32_t i = 0; i < nw; i++) {
      if (stackMap->get(i) == StackMap::Kind::AnyRef) {
        np += 1;
      }
    }
    MOZ_ASSERT(size_t(np) == countedPointers);
  }
#endif

  return true;
}

//////////////////////////////////////////////////////////////////////////////
//
// Stack frame methods.

void BaseStackFrame::zeroLocals(BaseRegAlloc* ra) {
  MOZ_ASSERT(varLow_ != UINT32_MAX);

  if (varLow_ == varHigh_) {
    return;
  }

  static const uint32_t wordSize = sizeof(void*);

  // The adjustments to 'low' by the size of the item being stored compensates
  // for the fact that locals offsets are the offsets from Frame to the bytes
  // directly "above" the locals in the locals area.  See comment at Local.

  // On 64-bit systems we may have 32-bit alignment for the local area as it
  // may be preceded by parameters and prologue/debug data.

  uint32_t low = varLow_;
  if (low % wordSize) {
    masm.store32(Imm32(0), Address(sp_, localOffset(low + 4)));
    low += 4;
  }
  MOZ_ASSERT(low % wordSize == 0);

  const uint32_t high = AlignBytes(varHigh_, wordSize);

  // An UNROLL_LIMIT of 16 is chosen so that we only need an 8-bit signed
  // immediate to represent the offset in the store instructions in the loop
  // on x64.

  const uint32_t UNROLL_LIMIT = 16;
  const uint32_t initWords = (high - low) / wordSize;
  const uint32_t tailWords = initWords % UNROLL_LIMIT;
  const uint32_t loopHigh = high - (tailWords * wordSize);

  // With only one word to initialize, just store an immediate zero.

  if (initWords == 1) {
    masm.storePtr(ImmWord(0), Address(sp_, localOffset(low + wordSize)));
    return;
  }

  // For other cases, it's best to have a zero in a register.
  //
  // One can do more here with SIMD registers (store 16 bytes at a time) or
  // with instructions like STRD on ARM (store 8 bytes at a time), but that's
  // for another day.

  RegI32 zero = ra->needI32();
  masm.mov(ImmWord(0), zero);

  // For the general case we want to have a loop body of UNROLL_LIMIT stores
  // and then a tail of less than UNROLL_LIMIT stores.  When initWords is less
  // than 2*UNROLL_LIMIT the loop trip count is at most 1 and there is no
  // benefit to having the pointer calculations and the compare-and-branch.
  // So we completely unroll when we have initWords < 2 * UNROLL_LIMIT.  (In
  // this case we'll end up using 32-bit offsets on x64 for up to half of the
  // stores, though.)

  // Fully-unrolled case.

  if (initWords < 2 * UNROLL_LIMIT) {
    for (uint32_t i = low; i < high; i += wordSize) {
      masm.storePtr(zero, Address(sp_, localOffset(i + wordSize)));
    }
    ra->freeI32(zero);
    return;
  }

  // Unrolled loop with a tail. Stores will use negative offsets. That's OK
  // for x86 and ARM, at least.

  // Compute pointer to the highest-addressed slot on the frame.
  RegI32 p = ra->needI32();
  masm.computeEffectiveAddress(Address(sp_, localOffset(low + wordSize)), p);

  // Compute pointer to the lowest-addressed slot on the frame that will be
  // initialized by the loop body.
  RegI32 lim = ra->needI32();
  masm.computeEffectiveAddress(Address(sp_, localOffset(loopHigh + wordSize)),
                               lim);

  // The loop body.  Eventually we'll have p == lim and exit the loop.
  Label again;
  masm.bind(&again);
  for (uint32_t i = 0; i < UNROLL_LIMIT; ++i) {
    masm.storePtr(zero, Address(p, -(wordSize * i)));
  }
  masm.subPtr(Imm32(UNROLL_LIMIT * wordSize), p);
  masm.branchPtr(Assembler::LessThan, lim, p, &again);

  // The tail.
  for (uint32_t i = 0; i < tailWords; ++i) {
    masm.storePtr(zero, Address(p, -(wordSize * i)));
  }

  ra->freeI32(p);
  ra->freeI32(lim);
  ra->freeI32(zero);
}

}  // namespace wasm
}  // namespace js