Skip to content

Commit b19be8d

Browse files
NattfarinnNattfarinn
authored
IBX-8562: Command to remove duplicated entries after faulty IBX-5388 fix
1 parent 9a62410 commit b19be8d

File tree

3 files changed

+268
-2
lines changed

3 files changed

+268
-2
lines changed

‎eZ/Bundle/EzPublishCoreBundle/Resources/config/commands.yml

+8
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,11 @@ services:
6262
$userHandler: '@ezpublish.spi.persistence.user_handler'
6363
tags:
6464
- { name: console.command }
65+
66+
Ibexa\Bundle\Core\Command\VirtualFieldDuplicateFixCommand:
67+
autowire: true
68+
autoconfigure: true
69+
arguments:
70+
$connection: '@ezpublish.persistence.connection'
71+
tags:
72+
- { name: console.command }

‎eZ/Bundle/EzPublishDebugBundle/Twig/DebugTemplate.php

+2-2
Original file line numberDiff line numberDiff line change
@@ -76,9 +76,9 @@ public function getSourceContext(): Source
7676
return new Source('', '');
7777
}
7878

79-
protected function doDisplay(array $context, array $blocks = []): string
79+
protected function doDisplay(array $context, array $blocks = []): iterable
8080
{
81-
return '';
81+
return [];
8282
}
8383

8484
/**

‎src/bundle/Core/Command/VirtualFieldDuplicateFixCommand.php

+258
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
<?php
2+
3+
/**
4+
* @copyright Copyright (C) Ibexa AS. All rights reserved.
5+
* @license For full copyright and license information view LICENSE file distributed with this source code.
6+
*/
7+
declare(strict_types=1);
8+
9+
namespace Ibexa\Bundle\Core\Command;
10+
11+
use Doctrine\DBAL\Connection;
12+
use Symfony\Component\Console\Command\Command;
13+
use Symfony\Component\Console\Input\InputInterface;
14+
use Symfony\Component\Console\Input\InputOption;
15+
use Symfony\Component\Console\Output\OutputInterface;
16+
use Symfony\Component\Console\Question\ConfirmationQuestion;
17+
use Symfony\Component\Console\Style\SymfonyStyle;
18+
use Symfony\Component\Stopwatch\Stopwatch;
19+
20+
final class VirtualFieldDuplicateFixCommand extends Command
21+
{
22+
private const DEFAULT_BATCH_SIZE = 10000;
23+
24+
private const MAX_ITERATIONS_UNLIMITED = -1;
25+
26+
private const DEFAULT_SLEEP = 0;
27+
28+
protected static $defaultName = 'ibexa:content:remove-duplicate-fields';
29+
30+
protected static $defaultDescription = 'Removes duplicate fields created as a result of faulty IBX-5388 performance fix.';
31+
32+
/** @var \Doctrine\DBAL\Connection */
33+
private $connection;
34+
35+
public function __construct(
36+
Connection $connection
37+
) {
38+
parent::__construct();
39+
40+
$this->connection = $connection;
41+
}
42+
43+
public function configure(): void
44+
{
45+
$this->addOption(
46+
'batch-size',
47+
'b',
48+
InputOption::VALUE_REQUIRED,
49+
'Number of attributes affected per iteration',
50+
self::DEFAULT_BATCH_SIZE
51+
);
52+
53+
$this->addOption(
54+
'max-iterations',
55+
'i',
56+
InputOption::VALUE_REQUIRED,
57+
'Max iterations count (default or -1: unlimited)',
58+
self::MAX_ITERATIONS_UNLIMITED
59+
);
60+
61+
$this->addOption(
62+
'sleep',
63+
's',
64+
InputOption::VALUE_REQUIRED,
65+
'Wait between iterations, in milliseconds',
66+
self::DEFAULT_SLEEP
67+
);
68+
}
69+
70+
protected function execute(InputInterface $input, OutputInterface $output): int
71+
{
72+
$style = new SymfonyStyle($input, $output);
73+
$stopwatch = new Stopwatch(true);
74+
$stopwatch->start('total', 'command');
75+
76+
$batchSize = (int)$input->getOption('batch-size');
77+
if ($batchSize === 0) {
78+
$style->warning('Batch size is set to 0. Nothing to do.');
79+
80+
return Command::INVALID;
81+
}
82+
83+
$maxIterations = (int)$input->getOption('max-iterations');
84+
if ($maxIterations === 0) {
85+
$style->warning('Max iterations is set to 0. Nothing to do.');
86+
87+
return Command::INVALID;
88+
}
89+
90+
$sleep = (int)$input->getOption('sleep');
91+
92+
$totalCount = $this->getDuplicatedAttributeTotalCount($style, $stopwatch);
93+
94+
if ($totalCount === 0) {
95+
$style->success('Database is clean of attribute duplicates. Nothing to do.');
96+
97+
return Command::SUCCESS;
98+
}
99+
100+
if ($input->isInteractive()) {
101+
$confirmation = $this->askForConfirmation($style);
102+
if (!$confirmation) {
103+
$style->info('Confirmation rejected. Terminating.');
104+
105+
return Command::FAILURE;
106+
}
107+
}
108+
109+
$iteration = 1;
110+
$totalDeleted = 0;
111+
do {
112+
$deleted = 0;
113+
$stopwatch->start('iteration', 'sql');
114+
115+
$attributes = $this->getDuplicatedAttributesBatch($batchSize);
116+
foreach ($attributes as $attribute) {
117+
$attributeIds = $this->getDuplicatedAttributeIds($attribute);
118+
119+
if (!empty($attributeIds)) {
120+
$iterationDeleted = $this->deleteAttributes($attributeIds);
121+
122+
$deleted += $iterationDeleted;
123+
$totalDeleted += $iterationDeleted;
124+
}
125+
}
126+
127+
$style->info(
128+
sprintf(
129+
'Iteration %d: Removed %d duplicate database rows (total removed this execution: %d). [Debug %s]',
130+
$iteration,
131+
$deleted,
132+
$totalDeleted,
133+
$stopwatch->stop('iteration')
134+
)
135+
);
136+
137+
if ($maxIterations !== self::MAX_ITERATIONS_UNLIMITED && ++$iteration > $maxIterations) {
138+
$style->warning('Max iterations count reached. Terminating.');
139+
140+
return self::SUCCESS;
141+
}
142+
143+
// Wait, if needed, before moving to next iteration
144+
usleep($sleep * 1000);
145+
} while ($batchSize === count($attributes));
146+
147+
$style->success(sprintf(
148+
'Operation successful. Removed total of %d duplicate database rows. [Debug %s]',
149+
$totalDeleted,
150+
$stopwatch->stop('total')
151+
));
152+
153+
return Command::SUCCESS;
154+
}
155+
156+
private function getDuplicatedAttributeTotalCount(
157+
SymfonyStyle $style,
158+
Stopwatch $stopwatch
159+
): int {
160+
$stopwatch->start('total_count', 'sql');
161+
$query = $this->connection->createQueryBuilder()
162+
->select('COUNT(a.id) as instances')
163+
->groupBy('version', 'contentclassattribute_id', 'contentobject_id', 'language_id')
164+
->from('ezcontentobject_attribute', 'a')
165+
->having('instances > 1');
166+
167+
$count = $query->execute()->rowCount();
168+
169+
if ($count > 0) {
170+
$style->warning(
171+
sprintf(
172+
'Found %d of affected attributes. [Debug: %s]',
173+
$count,
174+
$stopwatch->stop('total_count')
175+
)
176+
);
177+
}
178+
179+
return $count;
180+
}
181+
182+
/**
183+
* @phpstan-return array<array{
184+
* version: int,
185+
* contentclassattribute_id: int,
186+
* contentobject_id: int,
187+
* language_id: int,
188+
* }>
189+
*/
190+
private function getDuplicatedAttributesBatch(int $batchSize): array
191+
{
192+
$query = $this->connection->createQueryBuilder();
193+
194+
$query
195+
->select('version', 'contentclassattribute_id', 'contentobject_id', 'language_id')
196+
->groupBy('version', 'contentclassattribute_id', 'contentobject_id', 'language_id')
197+
->from('ezcontentobject_attribute')
198+
->having('COUNT(id) > 1')
199+
->setFirstResult(0)
200+
->setMaxResults($batchSize);
201+
202+
return $query->execute()->fetchAllAssociative();
203+
}
204+
205+
/**
206+
* @phpstan-param array{
207+
* version: int,
208+
* contentclassattribute_id: int,
209+
* contentobject_id: int,
210+
* language_id: int
211+
* } $attribute
212+
*
213+
* @return int[]
214+
*/
215+
private function getDuplicatedAttributeIds(array $attribute): array
216+
{
217+
$query = $this->connection->createQueryBuilder();
218+
219+
$query
220+
->select('id')
221+
->from('ezcontentobject_attribute')
222+
->andWhere('version = :version')
223+
->andWhere('contentclassattribute_id = :contentclassattribute_id')
224+
->andWhere('contentobject_id = :contentobject_id')
225+
->andWhere('language_id = :language_id')
226+
->orderBy('id', 'ASC')
227+
// Keep the original attribute row, the very first one
228+
->setFirstResult(1);
229+
230+
$query->setParameters($attribute);
231+
$result = $query->execute()->fetchFirstColumn();
232+
233+
return array_map('intval', $result);
234+
}
235+
236+
private function askForConfirmation(SymfonyStyle $style): bool
237+
{
238+
$style->warning('Operation is irreversible.');
239+
240+
return $style->askQuestion(
241+
new ConfirmationQuestion(
242+
'Proceed with deletion?',
243+
false
244+
)
245+
);
246+
}
247+
248+
private function deleteAttributes($ids): int
249+
{
250+
$query = $this->connection->createQueryBuilder();
251+
252+
$query
253+
->delete('ezcontentobject_attribute')
254+
->andWhere($query->expr()->in('id', $ids));
255+
256+
return (int)$query->execute();
257+
}
258+
}

0 commit comments

Comments
 (0)