WithChunkReadingTest.php 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602
  1. <?php
  2. namespace Maatwebsite\Excel\Tests\Concerns;
  3. use DateTime;
  4. use Exception;
  5. use Illuminate\Database\Eloquent\Model;
  6. use Illuminate\Support\Facades\DB;
  7. use Maatwebsite\Excel\Concerns\Importable;
  8. use Maatwebsite\Excel\Concerns\ToArray;
  9. use Maatwebsite\Excel\Concerns\ToModel;
  10. use Maatwebsite\Excel\Concerns\WithBatchInserts;
  11. use Maatwebsite\Excel\Concerns\WithChunkReading;
  12. use Maatwebsite\Excel\Concerns\WithEvents;
  13. use Maatwebsite\Excel\Concerns\WithFormatData;
  14. use Maatwebsite\Excel\Concerns\WithHeadingRow;
  15. use Maatwebsite\Excel\Concerns\WithMultipleSheets;
  16. use Maatwebsite\Excel\Events\AfterImport;
  17. use Maatwebsite\Excel\Events\BeforeImport;
  18. use Maatwebsite\Excel\Events\ImportFailed;
  19. use Maatwebsite\Excel\Reader;
  20. use Maatwebsite\Excel\Tests\Data\Stubs\Database\Group;
  21. use Maatwebsite\Excel\Tests\Data\Stubs\Database\User;
  22. use Maatwebsite\Excel\Tests\TestCase;
  23. use PhpOffice\PhpSpreadsheet\Shared\Date;
  24. use PHPUnit\Framework\Assert;
  25. use Throwable;
  26. class WithChunkReadingTest extends TestCase
  27. {
  28. /**
  29. * Setup the test environment.
  30. */
  31. protected function setUp(): void
  32. {
  33. parent::setUp();
  34. $this->loadLaravelMigrations(['--database' => 'testing']);
  35. $this->loadMigrationsFrom(dirname(__DIR__) . '/Data/Stubs/Database/Migrations');
  36. }
  37. /**
  38. * @test
  39. */
  40. public function can_import_to_model_in_chunks_un()
  41. {
  42. DB::connection()->enableQueryLog();
  43. $import = new class implements ToModel, WithChunkReading, WithEvents
  44. {
  45. use Importable;
  46. public $before = 0;
  47. public $after = 0;
  48. /**
  49. * @param array $row
  50. * @return Model|null
  51. */
  52. public function model(array $row)
  53. {
  54. return new User([
  55. 'name' => $row[0],
  56. 'email' => $row[1],
  57. 'password' => 'secret',
  58. ]);
  59. }
  60. /**
  61. * @return int
  62. */
  63. public function chunkSize(): int
  64. {
  65. return 1;
  66. }
  67. /**
  68. * @return array
  69. */
  70. public function registerEvents(): array
  71. {
  72. return [
  73. BeforeImport::class => function (BeforeImport $event) {
  74. Assert::assertInstanceOf(Reader::class, $event->reader);
  75. $this->before++;
  76. },
  77. AfterImport::class => function (AfterImport $event) {
  78. Assert::assertInstanceOf(Reader::class, $event->reader);
  79. $this->after++;
  80. },
  81. ];
  82. }
  83. };
  84. $import->import('import-users.xlsx');
  85. $this->assertCount(2, DB::getQueryLog());
  86. DB::connection()->disableQueryLog();
  87. $this->assertEquals(1, $import->before, 'BeforeImport was not called or more than once.');
  88. $this->assertEquals(1, $import->after, 'AfterImport was not called or more than once.');
  89. }
  90. /**
  91. * @test
  92. */
  93. public function can_import_to_model_in_chunks_and_insert_in_batches()
  94. {
  95. DB::connection()->enableQueryLog();
  96. $import = new class implements ToModel, WithChunkReading, WithBatchInserts
  97. {
  98. use Importable;
  99. /**
  100. * @param array $row
  101. * @return Model|null
  102. */
  103. public function model(array $row)
  104. {
  105. return new Group([
  106. 'name' => $row[0],
  107. ]);
  108. }
  109. /**
  110. * @return int
  111. */
  112. public function chunkSize(): int
  113. {
  114. return 1000;
  115. }
  116. /**
  117. * @return int
  118. */
  119. public function batchSize(): int
  120. {
  121. return 1000;
  122. }
  123. };
  124. $import->import('import-batches.xlsx');
  125. $this->assertCount(5000 / $import->batchSize(), DB::getQueryLog());
  126. DB::connection()->disableQueryLog();
  127. }
  128. /**
  129. * @test
  130. */
  131. public function can_import_to_model_in_chunks_and_insert_in_batches_with_heading_row()
  132. {
  133. DB::connection()->enableQueryLog();
  134. $import = new class implements ToModel, WithChunkReading, WithBatchInserts, WithHeadingRow
  135. {
  136. use Importable;
  137. /**
  138. * @param array $row
  139. * @return Model|null
  140. */
  141. public function model(array $row)
  142. {
  143. return new Group([
  144. 'name' => $row['name'],
  145. ]);
  146. }
  147. /**
  148. * @return int
  149. */
  150. public function chunkSize(): int
  151. {
  152. return 1000;
  153. }
  154. /**
  155. * @return int
  156. */
  157. public function batchSize(): int
  158. {
  159. return 1000;
  160. }
  161. };
  162. $import->import('import-batches-with-heading-row.xlsx');
  163. $this->assertCount(5000 / $import->batchSize(), DB::getQueryLog());
  164. DB::connection()->disableQueryLog();
  165. }
  166. /**
  167. * @test
  168. */
  169. public function can_import_csv_in_chunks_and_insert_in_batches()
  170. {
  171. DB::connection()->enableQueryLog();
  172. $import = new class implements ToModel, WithChunkReading, WithBatchInserts
  173. {
  174. use Importable;
  175. /**
  176. * @param array $row
  177. * @return Model|null
  178. */
  179. public function model(array $row)
  180. {
  181. return new Group([
  182. 'name' => $row[0],
  183. ]);
  184. }
  185. /**
  186. * @return int
  187. */
  188. public function chunkSize(): int
  189. {
  190. return 1000;
  191. }
  192. /**
  193. * @return int
  194. */
  195. public function batchSize(): int
  196. {
  197. return 1000;
  198. }
  199. };
  200. $import->import('import-batches.csv');
  201. $this->assertCount(10, DB::getQueryLog());
  202. DB::connection()->disableQueryLog();
  203. }
  204. /**
  205. * @test
  206. */
  207. public function can_import_to_model_in_chunks_and_insert_in_batches_with_multiple_sheets()
  208. {
  209. DB::connection()->enableQueryLog();
  210. $import = new class implements ToModel, WithChunkReading, WithBatchInserts
  211. {
  212. use Importable;
  213. /**
  214. * @param array $row
  215. * @return Model|null
  216. */
  217. public function model(array $row)
  218. {
  219. return new Group([
  220. 'name' => $row[0],
  221. ]);
  222. }
  223. /**
  224. * @return int
  225. */
  226. public function chunkSize(): int
  227. {
  228. return 1000;
  229. }
  230. /**
  231. * @return int
  232. */
  233. public function batchSize(): int
  234. {
  235. return 1000;
  236. }
  237. };
  238. $import->import('import-batches-multiple-sheets.xlsx');
  239. $this->assertCount(10, DB::getQueryLog());
  240. DB::connection()->disableQueryLog();
  241. }
  242. /**
  243. * @test
  244. */
  245. public function can_import_to_array_in_chunks()
  246. {
  247. $import = new class implements ToArray, WithChunkReading, WithFormatData
  248. {
  249. use Importable;
  250. public $called = 0;
  251. /**
  252. * @param array $array
  253. */
  254. public function array(array $array)
  255. {
  256. $this->called++;
  257. Assert::assertCount(100, $array);
  258. }
  259. /**
  260. * @return int
  261. */
  262. public function chunkSize(): int
  263. {
  264. return 100;
  265. }
  266. };
  267. $import->import('import-batches.xlsx');
  268. $this->assertEquals(50, $import->called);
  269. }
  270. /**
  271. * @test
  272. */
  273. public function can_import_to_model_in_chunks_and_insert_in_batches_with_multiple_sheets_objects_by_index()
  274. {
  275. DB::connection()->enableQueryLog();
  276. $import = new class implements WithMultipleSheets, WithChunkReading
  277. {
  278. use Importable;
  279. /**
  280. * @return int
  281. */
  282. public function chunkSize(): int
  283. {
  284. return 1000;
  285. }
  286. /**
  287. * @return array
  288. */
  289. public function sheets(): array
  290. {
  291. return [
  292. new class implements ToModel, WithBatchInserts
  293. {
  294. /**
  295. * @param array $row
  296. * @return Model|null
  297. */
  298. public function model(array $row)
  299. {
  300. return new Group([
  301. 'name' => $row[0],
  302. ]);
  303. }
  304. /**
  305. * @return int
  306. */
  307. public function batchSize(): int
  308. {
  309. return 1000;
  310. }
  311. },
  312. new class implements ToModel, WithBatchInserts
  313. {
  314. /**
  315. * @param array $row
  316. * @return Model|null
  317. */
  318. public function model(array $row)
  319. {
  320. return new Group([
  321. 'name' => $row[0],
  322. ]);
  323. }
  324. /**
  325. * @return int
  326. */
  327. public function batchSize(): int
  328. {
  329. return 2000;
  330. }
  331. },
  332. ];
  333. }
  334. };
  335. $import->import('import-batches-multiple-sheets.xlsx');
  336. $this->assertCount(10, DB::getQueryLog());
  337. DB::connection()->disableQueryLog();
  338. }
  339. /**
  340. * @test
  341. */
  342. public function can_import_to_model_in_chunks_and_insert_in_batches_with_multiple_sheets_objects_by_name()
  343. {
  344. DB::connection()->enableQueryLog();
  345. $import = new class implements WithMultipleSheets, WithChunkReading
  346. {
  347. use Importable;
  348. /**
  349. * @return int
  350. */
  351. public function chunkSize(): int
  352. {
  353. return 1000;
  354. }
  355. /**
  356. * @return array
  357. */
  358. public function sheets(): array
  359. {
  360. return [
  361. 'Worksheet' => new class implements ToModel, WithBatchInserts
  362. {
  363. /**
  364. * @param array $row
  365. * @return Model|null
  366. */
  367. public function model(array $row)
  368. {
  369. return new Group([
  370. 'name' => $row[0],
  371. ]);
  372. }
  373. /**
  374. * @return int
  375. */
  376. public function batchSize(): int
  377. {
  378. return 1000;
  379. }
  380. },
  381. 'Worksheet2' => new class implements ToModel, WithBatchInserts
  382. {
  383. /**
  384. * @param array $row
  385. * @return Model|null
  386. */
  387. public function model(array $row)
  388. {
  389. return new Group([
  390. 'name' => $row[0],
  391. ]);
  392. }
  393. /**
  394. * @return int
  395. */
  396. public function batchSize(): int
  397. {
  398. return 2000;
  399. }
  400. },
  401. ];
  402. }
  403. };
  404. $import->import('import-batches-multiple-sheets.xlsx');
  405. $this->assertCount(10, DB::getQueryLog());
  406. DB::connection()->disableQueryLog();
  407. }
  408. /**
  409. * @test
  410. */
  411. public function can_catch_job_failed_in_chunks()
  412. {
  413. $import = new class implements ToModel, WithChunkReading, WithEvents
  414. {
  415. use Importable;
  416. public $failed = false;
  417. /**
  418. * @param array $row
  419. * @return Model|null
  420. */
  421. public function model(array $row)
  422. {
  423. throw new Exception('Something went wrong in the chunk');
  424. }
  425. /**
  426. * @return int
  427. */
  428. public function chunkSize(): int
  429. {
  430. return 1;
  431. }
  432. /**
  433. * @return array
  434. */
  435. public function registerEvents(): array
  436. {
  437. return [
  438. ImportFailed::class => function (ImportFailed $event) {
  439. Assert::assertInstanceOf(Throwable::class, $event->getException());
  440. Assert::assertEquals('Something went wrong in the chunk', $event->e->getMessage());
  441. $this->failed = true;
  442. },
  443. ];
  444. }
  445. };
  446. try {
  447. $import->import('import-users.xlsx');
  448. } catch (Throwable $e) {
  449. $this->assertInstanceOf(Exception::class, $e);
  450. $this->assertEquals('Something went wrong in the chunk', $e->getMessage());
  451. }
  452. $this->assertTrue($import->failed, 'ImportFailed event was not called.');
  453. }
  454. /**
  455. * @test
  456. */
  457. public function can_import_to_array_and_format_in_chunks()
  458. {
  459. config()->set('excel.imports.read_only', false);
  460. $import = new class implements ToArray, WithChunkReading, WithFormatData
  461. {
  462. use Importable;
  463. /**
  464. * @param array $array
  465. */
  466. public function array(array $array)
  467. {
  468. Assert::assertCount(2, $array);
  469. Assert::assertCount(1, $array[0]);
  470. Assert::assertCount(1, $array[1]);
  471. Assert::assertIsString($array[0][0]);
  472. Assert::assertIsString($array[1][0]);
  473. Assert::assertEquals('01/12/22', $array[0][0]);
  474. Assert::assertEquals('2023-02-20', $array[1][0]);
  475. }
  476. /**
  477. * @return int
  478. */
  479. public function chunkSize(): int
  480. {
  481. return 2;
  482. }
  483. };
  484. $import->import('import-batches-with-date.xlsx');
  485. }
  486. /**
  487. * @test
  488. */
  489. public function can_import_to_array_in_chunks_without_formatting()
  490. {
  491. config()->set('excel.imports.read_only', true);
  492. $import = new class implements ToArray, WithChunkReading
  493. {
  494. use Importable;
  495. /**
  496. * @param array $array
  497. */
  498. public function array(array $array)
  499. {
  500. Assert::assertCount(2, $array);
  501. Assert::assertCount(1, $array[0]);
  502. Assert::assertCount(1, $array[1]);
  503. Assert::assertIsInt($array[0][0]);
  504. Assert::assertIsInt($array[1][0]);
  505. Assert::assertEquals((int) Date::dateTimeToExcel(DateTime::createFromFormat('Y-m-d', '2022-12-01')->setTime(0, 0, 0, 0)), $array[0][0]);
  506. Assert::assertEquals((int) Date::dateTimeToExcel(DateTime::createFromFormat('Y-m-d', '2023-02-20')->setTime(0, 0, 0, 0)), $array[1][0]);
  507. }
  508. /**
  509. * @return int
  510. */
  511. public function chunkSize(): int
  512. {
  513. return 2;
  514. }
  515. };
  516. $import->import('import-batches-with-date.xlsx');
  517. }
  518. }