diff --git a/libs/image/kis_inpaint_mask.cpp b/libs/image/kis_inpaint_mask.cpp index e15c9854d6..f5bd1e5cee 100644 --- a/libs/image/kis_inpaint_mask.cpp +++ b/libs/image/kis_inpaint_mask.cpp @@ -1,65 +1,64 @@ /* * Copyright (c) 2017 Eugene Ingerman * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "kis_inpaint_mask.h" #include "kis_debug.h" #include #include #include #include #include #include "kis_paint_device.h" #include "kis_painter.h" #include "kis_node_visitor.h" #include "kis_processing_visitor.h" #include "KoColorSpaceRegistry.h" KisInpaintMask::KisInpaintMask() - : KisTransparencyMask() + : KisTransparencyMask() { } KisInpaintMask::KisInpaintMask(const KisInpaintMask& rhs) - : KisTransparencyMask(rhs) + : KisTransparencyMask(rhs) { } KisInpaintMask::~KisInpaintMask() { } QRect KisInpaintMask::decorateRect(KisPaintDeviceSP &src, - KisPaintDeviceSP &dst, - const QRect & rc, - PositionToFilthy maskPos) const + KisPaintDeviceSP &dst, + const QRect & rc, + PositionToFilthy maskPos) const { Q_UNUSED(maskPos); KIS_ASSERT(dst != src); if (src != dst) { KisPainter::copyAreaOptimized(rc.topLeft(), src, dst, rc); src->fill(rc, KoColor(Qt::magenta, src->colorSpace())); } return rc; } - diff --git a/libs/image/kis_inpaint_mask.h b/libs/image/kis_inpaint_mask.h index e686214ad9..52cc2dfb69 100644 --- a/libs/image/kis_inpaint_mask.h +++ b/libs/image/kis_inpaint_mask.h @@ -1,49 +1,50 @@ /* * Copyright (c) 2017 Eugene Ingerman * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef _KIS_INPAINT_MASK_ #define _KIS_INPAINT_MASK_ #include "kis_types.h" #include "kis_transparency_mask.h" class QRect; /** * A inpaint mask is a single channel mask that works with inpaint operation to denote area affected by inpaint operation. * */ class KRITAIMAGE_EXPORT KisInpaintMask : public KisTransparencyMask { Q_OBJECT public: KisInpaintMask(); KisInpaintMask(const KisInpaintMask& rhs); virtual ~KisInpaintMask(); - KisNodeSP clone() const { + KisNodeSP clone() const + { return KisNodeSP(new KisInpaintMask(*this)); } QRect decorateRect(KisPaintDeviceSP &src, KisPaintDeviceSP &dst, const QRect & rc, PositionToFilthy maskPos) const; }; #endif //_KIS_INPAINT_MASK_ diff --git a/plugins/tools/tool_smart_patch/kis_inpaint.cpp b/plugins/tools/tool_smart_patch/kis_inpaint.cpp index b6e5772633..65b8826149 100644 --- a/plugins/tools/tool_smart_patch/kis_inpaint.cpp +++ b/plugins/tools/tool_smart_patch/kis_inpaint.cpp @@ -1,984 +1,985 @@ /* * Copyright (c) 2017 Eugene Ingerman * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ /** * Inpaint using the PatchMatch Algorithm * * | PatchMatch : A Randomized Correspondence Algorithm for Structural Image Editing * | by Connelly Barnes and Eli Shechtman and Adam Finkelstein and Dan B Goldman * | ACM Transactions on Graphics (Proc. SIGGRAPH), vol.28, aug-2009 * * Original author Xavier Philippeau * Code adopted from: David Chatting https://github.com/davidchatting/PatchMatch */ #include #include #include #include "kis_paint_device.h" #include "kis_painter.h" #include "kis_selection.h" #include "kis_debug.h" #include "kis_paint_device_debug_utils.h" //#include "kis_random_accessor_ng.h" #include #include #include #include "KoColor.h" #include "KoColorSpace.h" #include "KoChannelInfo.h" #include "KoMixColorsOp.h" #include "KoColorModelStandardIds.h" #include "KoColorSpaceRegistry.h" const int MAX_DIST = 65535; const quint8 MASK_SET = 0; const quint8 MASK_CLEAR = 255; void patchImage(KisPaintDeviceSP, KisPaintDeviceSP, int radius); class ImageView { protected: quint8* m_data; int m_imageWidth; int m_imageHeight; int m_pixelSize; public: void Init(quint8* _data, int _imageWidth, int _imageHeight, int _pixelSize) { m_data = _data; m_imageWidth = _imageWidth; m_imageHeight = _imageHeight; m_pixelSize = _pixelSize; } ImageView() : m_data(nullptr) { m_imageHeight = m_imageWidth = m_pixelSize = 0; } ImageView(quint8* _data, int _imageWidth, int _imageHeight, int _pixelSize) { Init(_data, _imageWidth, _imageHeight, _pixelSize); } quint8* operator()(int x, int y) const { Q_ASSERT(m_data); Q_ASSERT((x >= 0) && (x < m_imageWidth) && (y >= 0) && (y < m_imageHeight)); return (m_data + x * m_pixelSize + y * m_imageWidth * m_pixelSize); } ImageView& operator=(const ImageView& other) { if (this != &other) { if (other.num_bytes() != num_bytes()) { delete[] m_data; m_data = nullptr; //to preserve invariance if next line throws exception m_data = new quint8[other.num_bytes()]; } std::copy(other.data(), other.data() + other.num_bytes(), m_data); m_imageHeight = other.m_imageHeight; m_imageWidth = other.m_imageWidth; m_pixelSize = other.m_pixelSize; } return *this; } //move assignement operator ImageView& operator=(ImageView&& other) noexcept { if (this != &other) { delete[] m_data; m_data = nullptr; Init(other.data(), other.m_imageWidth, other.m_imageHeight, other.m_pixelSize); other.m_data = nullptr; } return *this; } virtual ~ImageView() {} //this class doesn't own m_data, so it ain't going to delete it either. quint8* data(void) const { return m_data; } inline int num_elements(void) const { return m_imageHeight * m_imageWidth; } inline int num_bytes(void) const { return m_imageHeight * m_imageWidth * m_pixelSize; } inline int pixel_size(void) const { return m_pixelSize; } - void saveToDevice(KisPaintDeviceSP outDev, QRect rect ) + void saveToDevice(KisPaintDeviceSP outDev, QRect rect) { Q_ASSERT(outDev->colorSpace()->pixelSize() == (quint32) m_pixelSize); outDev->writeBytes(m_data, rect); } void DebugDump(const QString& fnamePrefix) { QRect imSize(QPoint(0, 0), QSize(m_imageWidth, m_imageHeight)); const KoColorSpace* cs = (m_pixelSize == 1) ? KoColorSpaceRegistry::instance()->alpha8() : (m_pixelSize == 3) ? KoColorSpaceRegistry::instance()->colorSpace("RGB", "U8", "") : KoColorSpaceRegistry::instance()->colorSpace("RGBA", "U8", ""); KisPaintDeviceSP dbout = new KisPaintDevice(cs); saveToDevice(dbout, imSize); KIS_DUMP_DEVICE_2(dbout, imSize, fnamePrefix, "./"); } }; class ImageData : public ImageView { public: ImageData() : ImageView() {} void Init(int _imageWidth, int _imageHeight, int _pixelSize) { m_data = new quint8[ _imageWidth * _imageHeight * _pixelSize ]; ImageView::Init(m_data, _imageWidth, _imageHeight, _pixelSize); } ImageData(int _imageWidth, int _imageHeight, int _pixelSize) : ImageView() { Init(_imageWidth, _imageHeight, _pixelSize); } void Init(KisPaintDeviceSP imageDev, const QRect& imageSize) { const KoColorSpace* cs = imageDev->colorSpace(); m_pixelSize = cs->pixelSize(); m_data = new quint8[ imageSize.width()*imageSize.height()*cs->pixelSize() ]; imageDev->readBytes(m_data, imageSize.x(), imageSize.y(), imageSize.width(), imageSize.height()); ImageView::Init(m_data, imageSize.width(), imageSize.height(), m_pixelSize); } ImageData(KisPaintDeviceSP imageDev, const QRect& imageSize) : ImageView() { Init(imageDev, imageSize); } virtual ~ImageData() { delete[] m_data; //ImageData owns m_data, so it has to delete it } }; class MaskedImage : public KisShared { private: QRect imageSize; int nChannels; const KoColorSpace* cs; const KoColorSpace* csMask; ImageData maskData; ImageData imageData; void cacheImageSize(KisPaintDeviceSP imageDev) { imageSize = imageDev->exactBounds(); } void cacheImage(KisPaintDeviceSP imageDev) { Q_ASSERT(!imageSize.isEmpty() && imageSize.isValid()); cs = imageDev->colorSpace(); nChannels = cs->channelCount(); imageData.Init(imageDev, imageSize); } void cacheMask(KisPaintDeviceSP maskDev) { Q_ASSERT(!imageSize.isEmpty() && imageSize.isValid()); Q_ASSERT(maskDev->colorSpace()->pixelSize() == 1); csMask = maskDev->colorSpace(); maskData.Init(maskDev, imageSize); //hard threshold for the initial mask //may be optional. needs testing - std::for_each(maskData.data(), maskData.data() + maskData.num_bytes(), [](quint8& v){ v = (vwriteBytes(imageData.data(), 0, 0, W, H); maskDev->writeBytes(maskData.data(), 0, 0, W, H); ImageData newImage(newW, newH, cs->pixelSize()); ImageData newMask(newW, newH, 1); KoDummyUpdater updater; KisTransformWorker worker(imageDev, 1. / 2., 1. / 2., 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, &updater, KisFilterStrategyRegistry::instance()->value("Bicubic")); worker.run(); KisTransformWorker workerMask(maskDev, 1. / 2., 1. / 2., 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, &updater, KisFilterStrategyRegistry::instance()->value("Bicubic")); workerMask.run(); imageDev->readBytes(newImage.data(), 0, 0, newW, newH); maskDev->readBytes(newMask.data(), 0, 0, newW, newH); imageData = std::move(newImage); maskData = std::move(newMask); for (int i = 0; i < imageData.num_elements(); ++i) { quint8* maskPix = maskData.data() + i * maskData.pixel_size(); - if (*maskPix == MASK_SET ) { + if (*maskPix == MASK_SET) { for (int k = 0; k < imageData.pixel_size(); k++) *(imageData.data() + i * imageData.pixel_size() + k) = 0; } else { *maskPix = MASK_CLEAR; } } imageSize = QRect(0, 0, newW, newH); // int nmasked = countMasked(); // printf("Masked: %d size: %dx%d\n", nmasked, newW, newH); // maskData.DebugDump("maskData"); } void upscale(int newW, int newH) { int H = imageSize.height(); int W = imageSize.width(); ImageData newImage(newW, newH, cs->pixelSize()); ImageData newMask(newW, newH, 1); QVector colors(nChannels, 0.f); QVector v(nChannels, 0.f); for (int y = 0; y < newH; ++y) { for (int x = 0; x < newW; ++x) { // original pixel int xs = (x * W) / newW; int ys = (y * H) / newH; // copy to new image if (!isMasked(xs, ys)) { std::copy(imageData(xs, ys), imageData(xs, ys) + imageData.pixel_size(), newImage(x, y)); *newMask(x, y) = MASK_CLEAR; } else { std::fill(newImage(x, y), newImage(x, y) + newImage.pixel_size(), 0); *newMask(x, y) = MASK_SET; } } } imageData = std::move(newImage); maskData = std::move(newMask); imageSize = QRect(0, 0, newW, newH); } QRect size() { return imageSize; } KisSharedPtr copy(void) { KisSharedPtr clone = new MaskedImage(); clone->imageSize = this->imageSize; clone->nChannels = this->nChannels; clone->maskData = this->maskData; clone->imageData = this->imageData; clone->cs = this->cs; clone->csMask = this->csMask; return clone; } int countMasked(void) { int count = std::count_if(maskData.data(), maskData.data() + maskData.num_elements(), [](quint8 v) { return v < MASK_CLEAR; }); return count; } inline bool isMasked(int x, int y) { return (*maskData(x, y) < MASK_CLEAR); } //returns true if the patch contains a masked pixel bool containsMasked(int x, int y, int S) { for (int dy = -S; dy <= S; ++dy) { int ys = y + dy; if (ys < 0 || ys >= imageSize.height()) continue; for (int dx = -S; dx <= S; ++dx) { int xs = x + dx; if (xs < 0 || xs >= imageSize.width()) continue; if (isMasked(xs, ys)) return true; } } return false; } inline quint8 getImagePixelU8(int x, int y, int chan) const { return cs->scaleToU8(imageData(x, y), chan); } inline QVector getImagePixels(int x, int y) const { QVector v(cs->channelCount()); cs->normalisedChannelsValue(imageData(x, y), v); return v; } inline quint8* getImagePixel(int x, int y) { return imageData(x, y); } inline void setImagePixels(int x, int y, QVector& value) { cs->fromNormalisedChannelsValue(imageData(x, y), value); } inline void mixColors(std::vector< quint8* > pixels, std::vector< float > w, float wsum, quint8* dst) { const KoMixColorsOp* mixOp = cs->mixColorsOp(); size_t n = w.size(); assert(pixels.size() == n); std::vector< qint16 > weights; weights.clear(); float dif = 0; float scale = 255 / (wsum + 0.001); for (auto& v : w) { //compensated summation to increase accuracy float v1 = v * scale + dif; float v2 = std::round(v1); dif = v1 - v2; weights.push_back(v2); } mixOp->mixColors(pixels.data(), weights.data(), n, dst); } inline void setMask(int x, int y, quint8 v) { *(maskData(x, y)) = v; } inline int channelCount(void) const { return cs->channelCount(); } float distance(int x, int y, const MaskedImage& other, int xo, int yo) { float dsq = 0; quint32 nchannels = channelCount(); quint8* v1 = imageData(x, y); quint8* v2 = other.imageData(xo, yo); for (quint32 chan = 0; chan < nchannels; chan++) { //It's very important not to lose precision in the next line //TODO: This code only works for 8bpp data. Refactor to make it universal. float v = (float)(*((quint8*)v1 + chan)) - (float)(*((quint8*)v2 + chan)); dsq += v * v; } return dsq; } }; typedef KisSharedPtr MaskedImageSP; struct NNPixel { int x; int y; int distance; }; typedef boost::multi_array NNArray_type; struct Vote_elem { QVector channel_values; float w; }; typedef boost::multi_array Vote_type; class NearestNeighborField : public KisShared { private: template< typename T> T randomInt(T range) { return rand() % range; } //compute intial value of the distance term void initialize(void) { for (int y = 0; y < imSize.height(); y++) { for (int x = 0; x < imSize.width(); x++) { field[x][y].distance = distance(x, y, field[x][y].x, field[x][y].y); //if the distance is "infinity", try to find a better link int iter = 0; const int maxretry = 20; while (field[x][y].distance == MAX_DIST && iter < maxretry) { field[x][y].x = randomInt(imSize.width() + 1); field[x][y].y = randomInt(imSize.height() + 1); field[x][y].distance = distance(x, y, field[x][y].x, field[x][y].y); iter++; } } } } void init_similarity_curve(void) { float s_zero = 0.999; float t_halfmax = 0.10; float x = (s_zero - 0.5) * 2; float invtanh = 0.5 * std::log((1. + x) / (1. - x)); float coef = invtanh / t_halfmax; similarity.resize(MAX_DIST + 1); for (int i = 0; i < (int)similarity.size(); i++) { float t = (float)i / similarity.size(); similarity[i] = 0.5 - 0.5 * std::tanh(coef * (t - t_halfmax)); } } private: int patchSize; //patch size public: MaskedImageSP input; MaskedImageSP output; QRect imSize; NNArray_type field; std::vector similarity; quint32 nColors; QList channels; public: NearestNeighborField(const MaskedImageSP _input, MaskedImageSP _output, int _patchsize) : patchSize(_patchsize), input(_input), output(_output) { imSize = input->size(); field.resize(boost::extents[imSize.width()][imSize.height()]); init_similarity_curve(); nColors = input->channelCount(); //only color count, doesn't include alpha channels } void randomize(void) { for (int y = 0; y < imSize.height(); y++) { for (int x = 0; x < imSize.width(); x++) { field[x][y].x = randomInt(imSize.width() + 1); field[x][y].y = randomInt(imSize.height() + 1); field[x][y].distance = MAX_DIST; } } initialize(); } //initialize field from an existing (possibly smaller) nearest neighbor field void initialize(const NearestNeighborField& nnf) { float xscale = imSize.width() / nnf.imSize.width(); float yscale = imSize.height() / nnf.imSize.height(); for (int y = 0; y < imSize.height(); y++) { for (int x = 0; x < imSize.width(); x++) { int xlow = std::min((int)(x / xscale), nnf.imSize.width() - 1); int ylow = std::min((int)(y / yscale), nnf.imSize.height() - 1); field[x][y].x = nnf.field[xlow][ylow].x * xscale; field[x][y].y = nnf.field[xlow][ylow].y * yscale; field[x][y].distance = MAX_DIST; } } initialize(); } //multi-pass NN-field minimization (see "PatchMatch" - page 4) void minimize(int pass) { int min_x = 0; int min_y = 0; int max_x = imSize.width() - 1; int max_y = imSize.height() - 1; for (int i = 0; i < pass; i++) { //scanline order for (int y = min_y; y < max_y; y++) for (int x = min_x; x <= max_x; x++) if (field[x][y].distance > 0) minimizeLink(x, y, 1); //reverse scanline order for (int y = max_y; y >= min_y; y--) for (int x = max_x; x >= min_x; x--) if (field[x][y].distance > 0) minimizeLink(x, y, -1); } } void minimizeLink(int x, int y, int dir) { int xp, yp, dp; //Propagation Left/Right if (x - dir > 0 && x - dir < imSize.width()) { xp = field[x - dir][y].x + dir; yp = field[x - dir][y].y; dp = distance(x, y, xp, yp); if (dp < field[x][y].distance) { field[x][y].x = xp; field[x][y].y = yp; field[x][y].distance = dp; } } //Propagation Up/Down if (y - dir > 0 && y - dir < imSize.height()) { xp = field[x][y - dir].x; yp = field[x][y - dir].y + dir; dp = distance(x, y, xp, yp); if (dp < field[x][y].distance) { field[x][y].x = xp; field[x][y].y = yp; field[x][y].distance = dp; } } //Random search int wi = std::max(output->size().width(), output->size().height()); int xpi = field[x][y].x; int ypi = field[x][y].y; while (wi > 0) { xp = xpi + randomInt(2 * wi) - wi; yp = ypi + randomInt(2 * wi) - wi; xp = std::max(0, std::min(output->size().width() - 1, xp)); yp = std::max(0, std::min(output->size().height() - 1, yp)); dp = distance(x, y, xp, yp); if (dp < field[x][y].distance) { field[x][y].x = xp; field[x][y].y = yp; field[x][y].distance = dp; } wi /= 2; } } //compute distance between two patches int distance(int x, int y, int xp, int yp) { float distance = 0; float wsum = 0; float ssdmax = nColors * 255 * 255; //for each pixel in the source patch for (int dy = -patchSize; dy <= patchSize; dy++) { for (int dx = -patchSize; dx <= patchSize; dx++) { wsum += ssdmax; int xks = x + dx; int yks = y + dy; if (xks < 0 || xks >= input->size().width()) { distance += ssdmax; continue; } if (yks < 0 || yks >= input->size().height()) { distance += ssdmax; continue; } //cannot use masked pixels as a valid source of information if (input->isMasked(xks, yks)) { distance += ssdmax; continue; } //corresponding pixel in target patch int xkt = xp + dx; int ykt = yp + dy; if (xkt < 0 || xkt >= output->size().width()) { distance += ssdmax; continue; } if (ykt < 0 || ykt >= output->size().height()) { distance += ssdmax; continue; } //cannot use masked pixels as a valid source of information if (output->isMasked(xkt, ykt)) { distance += ssdmax; continue; } //SSD distance between pixels float ssd = input->distance(xks, yks, *output, xkt, ykt); //long ssd = input->distance(xks, yks, *input, xkt, ykt); distance += ssd; } } return (int)(MAX_DIST * (distance / wsum)); } static MaskedImageSP ExpectationMaximization(KisSharedPtr TargetToSource, int level, int radius, QList& pyramid); static void ExpectationStep(KisSharedPtr nnf, MaskedImageSP source, MaskedImageSP target, bool upscale); void EM_Step(MaskedImageSP source, MaskedImageSP target, int R, bool upscaled); }; typedef KisSharedPtr NearestNeighborFieldSP; class Inpaint { private: KisPaintDeviceSP devCache; MaskedImageSP initial; NearestNeighborFieldSP nnf_TargetToSource; NearestNeighborFieldSP nnf_SourceToTarget; int radius; QList pyramid; public: Inpaint(KisPaintDeviceSP dev, KisPaintDeviceSP devMask, int _radius) { initial = new MaskedImage(dev, devMask); radius = _radius; devCache = dev; } MaskedImageSP patch(void); MaskedImageSP patch_simple(void); }; MaskedImageSP Inpaint::patch() { MaskedImageSP source = initial->copy(); pyramid.append(initial); QRect size = source->size(); //qDebug() << "countMasked: " << source->countMasked() << "\n"; while ((size.width() > radius) && (size.height() > radius) && source->countMasked() > 0) { source->downsample2x(); //source->DebugDump("Pyramid"); //qDebug() << "countMasked1: " << source->countMasked() << "\n"; pyramid.append(source->copy()); size = source->size(); } int maxlevel = pyramid.size(); //qDebug() << "MaxLevel: " << maxlevel << "\n"; // The initial target is the same as the smallest source. // We consider that this target contains no masked pixels MaskedImageSP target = source->copy(); target->clearMask(); //recursively building nearest neighbor field for (int level = maxlevel - 1; level > 0; level--) { source = pyramid.at(level); if (level == maxlevel - 1) { //random initial guess nnf_TargetToSource = new NearestNeighborField(target, source, radius); nnf_TargetToSource->randomize(); } else { // then, we use the rebuilt (upscaled) target // and reuse the previous NNF as initial guess NearestNeighborFieldSP new_nnf_rev = new NearestNeighborField(target, source, radius); new_nnf_rev->initialize(*nnf_TargetToSource); nnf_TargetToSource = new_nnf_rev; } //Build an upscaled target by EM-like algorithm (see "PatchMatch" - page 6) target = NearestNeighborField::ExpectationMaximization(nnf_TargetToSource, level, radius, pyramid); //target->DebugDump( "target" ); } return target; } //EM-Like algorithm (see "PatchMatch" - page 6) //Returns a float sized target image MaskedImageSP NearestNeighborField::ExpectationMaximization(NearestNeighborFieldSP nnf_TargetToSource, int level, int radius, QList& pyramid) { int iterEM = std::min(2 * level, 4); int iterNNF = std::min(5, 1 + level); MaskedImageSP source = nnf_TargetToSource->output; MaskedImageSP target = nnf_TargetToSource->input; MaskedImageSP newtarget = nullptr; //EM loop for (int emloop = 1; emloop <= iterEM; emloop++) { //set the new target as current target if (!newtarget.isNull()) { nnf_TargetToSource->input = newtarget; target = newtarget; newtarget = nullptr; } for (int x = 0; x < target->size().width(); ++x) { for (int y = 0; y < target->size().height(); ++y) { if (!source->containsMasked(x, y, radius)) { nnf_TargetToSource->field[x][y].x = x; nnf_TargetToSource->field[x][y].y = y; nnf_TargetToSource->field[x][y].distance = 0; } } } //minimize the NNF nnf_TargetToSource->minimize(iterNNF); //Now we rebuild the target using best patches from source MaskedImageSP newsource = nullptr; bool upscaled = false; // Instead of upsizing the final target, we build the last target from the next level source image // So the final target is less blurry (see "Space-Time Video Completion" - page 5) if (level >= 1 && (emloop == iterEM)) { newsource = pyramid.at(level - 1); QRect sz = newsource->size(); newtarget = target->copy(); newtarget->upscale(sz.width(), sz.height()); upscaled = true; } else { newsource = pyramid.at(level); newtarget = target->copy(); upscaled = false; } //EM Step //EM_Step(newsource, newtarget, radius, upscaled); ExpectationStep(nnf_TargetToSource, newsource, newtarget, upscaled); } return newtarget; } void NearestNeighborField::ExpectationStep(NearestNeighborFieldSP nnf, MaskedImageSP source, MaskedImageSP target, bool upscale) { //int*** field = nnf->field; int R = nnf->patchSize; if (upscale) R *= 2; int H_nnf = nnf->input->size().height(); int W_nnf = nnf->input->size().width(); int H_target = target->size().height(); int W_target = target->size().width(); int H_source = source->size().height(); int W_source = source->size().width(); std::vector< quint8* > pixels; std::vector< float > weights; pixels.reserve(R * R); weights.reserve(R * R); for (int x = 0 ; x < W_target ; ++x) { for (int y = 0 ; y < H_target; ++y) { float wsum = 0; pixels.clear(); weights.clear(); if (!source->containsMasked(x, y, R + 4) /*&& upscale*/) { //speedup computation by copying parts that are not masked. pixels.push_back(source->getImagePixel(x, y)); weights.push_back(1.f); target->mixColors(pixels, weights, 1.f, target->getImagePixel(x, y)); } else { for (int dx = -R ; dx <= R; ++dx) { for (int dy = -R ; dy <= R ; ++dy) { // xpt,ypt = center pixel of the target patch int xpt = x + dx; int ypt = y + dy; int xst, yst; float w; if (!upscale) { if (xpt < 0 || xpt >= W_nnf || ypt < 0 || ypt >= H_nnf) continue; xst = nnf->field[xpt][ypt].x; yst = nnf->field[xpt][ypt].y; float dp = nnf->field[xpt][ypt].distance; // similarity measure between the two patches w = nnf->similarity[dp]; } else { if (xpt < 0 || (xpt / 2) >= W_nnf || ypt < 0 || (ypt / 2) >= H_nnf) continue; xst = 2 * nnf->field[xpt / 2][ypt / 2].x + (xpt % 2); yst = 2 * nnf->field[xpt / 2][ypt / 2].y + (ypt % 2); float dp = nnf->field[xpt / 2][ypt / 2].distance; // similarity measure between the two patches w = nnf->similarity[dp]; } int xs = xst - dx; int ys = yst - dy; if (xs < 0 || xs >= W_source || ys < 0 || ys >= H_source) continue; if (source->isMasked(xs, ys)) continue; pixels.push_back(source->getImagePixel(xs, ys)); weights.push_back(w); wsum += w; } } if (wsum < 1) continue; target->mixColors(pixels, weights, wsum, target->getImagePixel(x, y)); } } } } -QRect getMaskBoundingBox( KisPaintDeviceSP maskDev ) +QRect getMaskBoundingBox(KisPaintDeviceSP maskDev) { KoColor defaultMaskPixel = maskDev->defaultPixel(); - maskDev->setDefaultPixel( KoColor(Qt::white, maskDev->colorSpace())); + maskDev->setDefaultPixel(KoColor(Qt::white, maskDev->colorSpace())); QRect maskRect = maskDev->nonDefaultPixelArea(); - maskDev->setDefaultPixel( defaultMaskPixel ); + maskDev->setDefaultPixel(defaultMaskPixel); return maskRect; } QRect patchImage(KisPaintDeviceSP imageDev, KisPaintDeviceSP maskDev, int patchRadius, int accuracy) { - QRect maskRect = getMaskBoundingBox( maskDev ); + QRect maskRect = getMaskBoundingBox(maskDev); QRect imageRect = imageDev->exactBounds(); float scale = 1 + (accuracy / 25); //higher accuracy means we include more surrouding area around the patch. Minimum 2x padding. - int dx = maskRect.width()*scale; - int dy = maskRect.height()*scale; + int dx = maskRect.width() * scale; + int dy = maskRect.height() * scale; maskRect.adjust(-dx, -dy, dx, dy); maskRect = maskRect.intersected(imageRect); - KisPaintDeviceSP tempImageDev = new KisPaintDevice( imageDev->colorSpace() ); - KisPaintDeviceSP tempMaskDev = new KisPaintDevice( maskDev->colorSpace() ); - tempImageDev->makeCloneFrom( imageDev, maskRect ); - tempMaskDev->makeCloneFrom( maskDev, maskRect ); + KisPaintDeviceSP tempImageDev = new KisPaintDevice(imageDev->colorSpace()); + KisPaintDeviceSP tempMaskDev = new KisPaintDevice(maskDev->colorSpace()); + tempImageDev->makeCloneFrom(imageDev, maskRect); + tempMaskDev->makeCloneFrom(maskDev, maskRect); - if( !maskRect.isEmpty() ){ + if (!maskRect.isEmpty()) { Inpaint inpaint(tempImageDev, tempMaskDev, patchRadius); MaskedImageSP output = inpaint.patch(); - output->toPaintDevice( imageDev, maskRect ); + output->toPaintDevice(imageDev, maskRect); } return maskRect; } - diff --git a/plugins/tools/tool_smart_patch/kis_tool_smart_patch.cpp b/plugins/tools/tool_smart_patch/kis_tool_smart_patch.cpp index 5ef35e1ffc..13e6b67c87 100644 --- a/plugins/tools/tool_smart_patch/kis_tool_smart_patch.cpp +++ b/plugins/tools/tool_smart_patch/kis_tool_smart_patch.cpp @@ -1,238 +1,236 @@ /* * Copyright (c) 2017 Eugene Ingerman * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "kis_tool_smart_patch.h" #include "QApplication" #include #include #include #include "kis_canvas2.h" #include "kis_cursor.h" #include "kis_config.h" #include "kundo2magicstring.h" #include "KoProperties.h" #include "KoColorSpaceRegistry.h" #include "KoShapeController.h" #include "KoDocumentResourceManager.h" #include "kis_node_manager.h" #include "kis_cursor.h" #include "kis_tool_smart_patch_options_widget.h" #include "libs/image/kis_paint_device_debug_utils.h" #include "kis_resources_snapshot.h" #include "kis_layer.h" #include "kis_transaction.h" #include "kis_paint_layer.h" #include "kis_inpaint_mask.h" QRect patchImage(KisPaintDeviceSP imageDev, KisPaintDeviceSP maskDev, int radius, int accuracy); -struct KisToolSmartPatch::Private -{ +struct KisToolSmartPatch::Private { KisMaskSP mask = nullptr; KisNodeSP maskNode = nullptr; KisNodeSP paintNode = nullptr; KisPaintDeviceSP imageDev = nullptr; KisPaintDeviceSP maskDev = nullptr; KisResourcesSnapshotSP resources = nullptr; KoColor currentFgColor; KisToolSmartPatchOptionsWidget *optionsWidget = nullptr; }; KisToolSmartPatch::KisToolSmartPatch(KoCanvasBase * canvas) : KisToolFreehand(canvas, KisCursor::load("tool_freehand_cursor.png", 5, 5), kundo2_i18n("Smart Patch Stroke")), m_d(new Private) { setObjectName("tool_SmartPatch"); } KisToolSmartPatch::~KisToolSmartPatch() { m_d->optionsWidget = nullptr; } void KisToolSmartPatch::activate(ToolActivation activation, const QSet &shapes) { KisToolFreehand::activate(activation, shapes); } void KisToolSmartPatch::deactivate() { KisToolFreehand::deactivate(); } void KisToolSmartPatch::resetCursorStyle() { KisToolFreehand::resetCursorStyle(); } bool KisToolSmartPatch::canCreateInpaintMask() const { KisNodeSP node = currentNode(); return node && node->inherits("KisPaintLayer"); } QRect KisToolSmartPatch::inpaintImage(KisPaintDeviceSP maskDev, KisPaintDeviceSP imageDev) { int accuracy = 50; //default accuracy - middle value int patchRadius = 4; //default radius, which works well for most cases tested - if( !m_d.isNull() && m_d->optionsWidget ){ + if (!m_d.isNull() && m_d->optionsWidget) { accuracy = m_d->optionsWidget->getAccuracy(); patchRadius = m_d->optionsWidget->getPatchRadius(); } - return patchImage( imageDev, maskDev, patchRadius, accuracy ); + return patchImage(imageDev, maskDev, patchRadius, accuracy); } void KisToolSmartPatch::activatePrimaryAction() { KisToolFreehand::activatePrimaryAction(); } void KisToolSmartPatch::deactivatePrimaryAction() { KisToolFreehand::deactivatePrimaryAction(); } -void KisToolSmartPatch::createInpaintMask( void ) +void KisToolSmartPatch::createInpaintMask(void) { m_d->mask = new KisInpaintMask(); KisLayerSP parentLayer = qobject_cast(m_d->paintNode.data()); m_d->mask->initSelection(parentLayer); - image()->addNode( m_d->mask, m_d->paintNode ); + image()->addNode(m_d->mask, m_d->paintNode); } -void KisToolSmartPatch::deleteInpaintMask( void ) +void KisToolSmartPatch::deleteInpaintMask(void) { KisCanvas2 * kiscanvas = static_cast(canvas()); KisViewManager* viewManager = kiscanvas->viewManager(); - if( ! m_d->paintNode.isNull() ) - viewManager->nodeManager()->slotNonUiActivatedNode( m_d->paintNode ); + if (! m_d->paintNode.isNull()) + viewManager->nodeManager()->slotNonUiActivatedNode(m_d->paintNode); image()->removeNode(m_d->mask); m_d->mask = nullptr; } void KisToolSmartPatch::beginPrimaryAction(KoPointerEvent *event) { m_d->paintNode = currentNode(); KisCanvas2 * kiscanvas = static_cast(canvas()); KisViewManager* viewManager = kiscanvas->viewManager(); //we can only apply inpaint operation to paint layer - if ( !m_d->paintNode.isNull() && m_d->paintNode->inherits("KisPaintLayer") ){ + if (!m_d->paintNode.isNull() && m_d->paintNode->inherits("KisPaintLayer")) { if (!m_d->mask.isNull()) { viewManager->nodeManager()->slotNonUiActivatedNode(m_d->mask); } else { createInpaintMask(); viewManager->nodeManager()->slotNonUiActivatedNode(m_d->mask); //Collapse freehand drawing of the mask followed by inpaint operation into a single undo node canvas()->shapeController()->resourceManager()->undoStack()->beginMacro(kundo2_i18n("Smart Patch")); //User will be drawing on an alpha mask. Show color matching inpaint mask color. m_d->currentFgColor = canvas()->resourceManager()->foregroundColor(); canvas()->resourceManager()->setForegroundColor(KoColor(Qt::magenta, image()->colorSpace())); } KisToolFreehand::beginPrimaryAction(event); } else { viewManager-> - showFloatingMessage( - i18n("Select paint layer to use this tool"), - QIcon(), 2000, KisFloatingMessage::Medium, Qt::AlignCenter); + showFloatingMessage( + i18n("Select paint layer to use this tool"), + QIcon(), 2000, KisFloatingMessage::Medium, Qt::AlignCenter); } } void KisToolSmartPatch::continuePrimaryAction(KoPointerEvent *event) { if (!m_d->mask.isNull()) KisToolFreehand::continuePrimaryAction(event); } void KisToolSmartPatch::endPrimaryAction(KoPointerEvent *event) { - if( mode() != KisTool::PAINT_MODE ) + if (mode() != KisTool::PAINT_MODE) return; - if( m_d->mask.isNull() ) + if (m_d->mask.isNull()) return; KisToolFreehand::endPrimaryAction(event); //Next line is important. We need to wait for the paint operation to finish otherwise //mask will be incomplete. image()->waitForDone(); //User drew a mask on the temporary inpaint mask layer. Get this mask to pass to the inpaint algorithm m_d->maskDev = new KisPaintDevice(KoColorSpaceRegistry::instance()->alpha8()); - if( !m_d->mask.isNull() ){ + if (!m_d->mask.isNull()) { m_d->maskDev->makeCloneFrom(m_d->mask->paintDevice(), m_d->mask->paintDevice()->extent()); //Once we get the mask we delete the temporary layer deleteInpaintMask(); image()->waitForDone(); m_d->imageDev = currentNode()->paintDevice(); KisTransaction inpaintTransaction(kundo2_i18n("Inpaint Operation"), m_d->imageDev); QApplication::setOverrideCursor(KisCursor::waitCursor()); //actual inpaint operation - QRect changedRect = inpaintImage( m_d->maskDev, m_d->imageDev ); - currentNode()->setDirty( changedRect ); + QRect changedRect = inpaintImage(m_d->maskDev, m_d->imageDev); + currentNode()->setDirty(changedRect); inpaintTransaction.commit(image()->undoAdapter()); //Matching endmacro for inpaint operation canvas()->shapeController()->resourceManager()->undoStack()->endMacro(); QApplication::restoreOverrideCursor(); canvas()->resourceManager()->setForegroundColor(m_d->currentFgColor); } } QWidget * KisToolSmartPatch::createOptionWidget() { KisCanvas2 * kiscanvas = dynamic_cast(canvas()); m_d->optionsWidget = new KisToolSmartPatchOptionsWidget(kiscanvas->viewManager()->resourceProvider(), 0); m_d->optionsWidget->setObjectName(toolId() + "option widget"); return m_d->optionsWidget; } - diff --git a/plugins/tools/tool_smart_patch/kis_tool_smart_patch.h b/plugins/tools/tool_smart_patch/kis_tool_smart_patch.h index edf7b92f87..8797bb38b9 100644 --- a/plugins/tools/tool_smart_patch/kis_tool_smart_patch.h +++ b/plugins/tools/tool_smart_patch/kis_tool_smart_patch.h @@ -1,101 +1,103 @@ /* * Copyright (c) 2017 Eugene Ingerman * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef KIS_TOOL_SMART_PATCH_H_ #define KIS_TOOL_SMART_PATCH_H_ #include #include "kis_tool_freehand.h" #include "KoToolFactoryBase.h" #include #include #include #include #include #include class KActionCollection; class KoCanvasBase; class KisToolSmartPatch : public KisToolFreehand { Q_OBJECT public: KisToolSmartPatch(KoCanvasBase * canvas); virtual ~KisToolSmartPatch(); QWidget * createOptionWidget(); void activatePrimaryAction(); void deactivatePrimaryAction(); void beginPrimaryAction(KoPointerEvent *event); void continuePrimaryAction(KoPointerEvent *event); void endPrimaryAction(KoPointerEvent *event); protected Q_SLOTS: void resetCursorStyle(); public Q_SLOTS: virtual void activate(ToolActivation toolActivation, const QSet &shapes); void deactivate(); Q_SIGNALS: private: bool canCreateInpaintMask() const; - QRect inpaintImage( KisPaintDeviceSP maskDev, KisPaintDeviceSP imageDev ); + QRect inpaintImage(KisPaintDeviceSP maskDev, KisPaintDeviceSP imageDev); private: struct Private; const QScopedPointer m_d; void createInpaintMask(); void deleteInpaintMask(); }; class KisToolSmartPatchFactory : public KoToolFactoryBase { public: KisToolSmartPatchFactory() - : KoToolFactoryBase("KritaShape/KisToolSmartPatch") { + : KoToolFactoryBase("KritaShape/KisToolSmartPatch") + { setToolTip(i18n("Smart Patch Tool")); setSection(TOOL_TYPE_FILL); setIconName(koIconNameCStr("krita_tool_smart_patch")); setShortcut(QKeySequence(Qt::Key_Shift + Qt::Key_I)); setPriority(4); setActivationShapeId(KRITA_TOOL_ACTIVATION_ID); } virtual ~KisToolSmartPatchFactory() {} - virtual KoToolBase * createTool(KoCanvasBase *canvas) { + virtual KoToolBase * createTool(KoCanvasBase *canvas) + { return new KisToolSmartPatch(canvas); } }; #endif // KIS_TOOL_SMART_PATCH_H_ diff --git a/plugins/tools/tool_smart_patch/kis_tool_smart_patch_options_widget.cpp b/plugins/tools/tool_smart_patch/kis_tool_smart_patch_options_widget.cpp index 58a704585f..6f768c2762 100644 --- a/plugins/tools/tool_smart_patch/kis_tool_smart_patch_options_widget.cpp +++ b/plugins/tools/tool_smart_patch/kis_tool_smart_patch_options_widget.cpp @@ -1,68 +1,70 @@ /* * Copyright (c) 2017 Eugene Ingerman * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "kis_tool_smart_patch_options_widget.h" #include "ui_kis_tool_smart_patch_options_widget.h" #include #include "KisPaletteModel.h" #include "kis_config.h" #include #include "kis_canvas_resource_provider.h" -struct KisToolSmartPatchOptionsWidget::Private -{ +struct KisToolSmartPatchOptionsWidget::Private { Private() { } Ui_KisToolSmartPatchOptionsWidget *ui; - int getPatchRadius(void){ + int getPatchRadius(void) + { return ui->patchRadius->value(); } - int getAccuracy(void){ + int getAccuracy(void) + { return ui->accuracySlider->value(); } }; KisToolSmartPatchOptionsWidget::KisToolSmartPatchOptionsWidget(KisCanvasResourceProvider *provider, QWidget *parent) : QWidget(parent), m_d(new Private) { m_d->ui = new Ui_KisToolSmartPatchOptionsWidget(); m_d->ui->setupUi(this); } KisToolSmartPatchOptionsWidget::~KisToolSmartPatchOptionsWidget() { } -int KisToolSmartPatchOptionsWidget::getPatchRadius(){ +int KisToolSmartPatchOptionsWidget::getPatchRadius() +{ return m_d->getPatchRadius(); } -int KisToolSmartPatchOptionsWidget::getAccuracy(){ +int KisToolSmartPatchOptionsWidget::getAccuracy() +{ return m_d->getAccuracy(); } - diff --git a/plugins/tools/tool_smart_patch/kis_tool_smart_patch_options_widget.h b/plugins/tools/tool_smart_patch/kis_tool_smart_patch_options_widget.h index 023895c951..87289a0fc5 100644 --- a/plugins/tools/tool_smart_patch/kis_tool_smart_patch_options_widget.h +++ b/plugins/tools/tool_smart_patch/kis_tool_smart_patch_options_widget.h @@ -1,47 +1,47 @@ /* * Copyright (c) 2017 Eugene Ingerman * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef __KIS_TOOL_SMART_PATCH_OPTIONS_WIDGET_H #define __KIS_TOOL_SMART_PATCH_OPTIONS_WIDGET_H #include #include #include #include "kis_types.h" class KisCanvasResourceProvider; class KoColor; class KisToolSmartPatchOptionsWidget : public QWidget { Q_OBJECT public: KisToolSmartPatchOptionsWidget(KisCanvasResourceProvider *provider, QWidget *parent); ~KisToolSmartPatchOptionsWidget(); - int getPatchRadius( void ); - int getAccuracy( void ); + int getPatchRadius(void); + int getAccuracy(void); private: struct Private; const QScopedPointer m_d; }; #endif /* __KIS_TOOL_SMART_PATCH_OPTIONS_WIDGET_H */ diff --git a/plugins/tools/tool_smart_patch/tool_smartpatch.cpp b/plugins/tools/tool_smart_patch/tool_smartpatch.cpp index 7d5a939c4b..9539ce78b6 100644 --- a/plugins/tools/tool_smart_patch/tool_smartpatch.cpp +++ b/plugins/tools/tool_smart_patch/tool_smartpatch.cpp @@ -1,45 +1,45 @@ /* * Copyright (c) 2017 Eugene Ingerman * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "tool_smartpatch.h" #include #include #include #include #include #include "kis_paint_device.h" #include "kis_tool_smart_patch.h" K_PLUGIN_FACTORY_WITH_JSON(DefaultToolsFactory, "kritatoolsmartpatch.json", registerPlugin();) ToolSmartPatch::ToolSmartPatch(QObject *parent, const QVariantList &) - : QObject(parent) + : QObject(parent) { KoToolRegistry::instance()->add(new KisToolSmartPatchFactory()); } ToolSmartPatch::~ToolSmartPatch() { } #include "tool_smartpatch.moc"